evalscope 0.17.1__py3-none-any.whl → 1.0.0__py3-none-any.whl

evalscope/benchmarks/cmmlu/cmmlu_adapter.py

@@ -1,36 +1,16 @@
 # Copyright (c) Alibaba, Inc. and its affiliates.
 
-import csv
-import os
-from collections import defaultdict
-
-from evalscope.benchmarks import Benchmark, DataAdapter
-from evalscope.constants import EvalType, OutputType
-from evalscope.metrics import exact_match
-from evalscope.metrics.completion_parsers import ResponseParser
-from evalscope.utils.io_utils import csv_to_list
+from evalscope.api.benchmark import BenchmarkMeta, MultiChoiceAdapter
+from evalscope.api.dataset import Sample
+from evalscope.api.registry import register_benchmark
+from evalscope.constants import Tags
 from evalscope.utils.logger import get_logger
+from evalscope.utils.multi_choices import MultipleChoiceTemplate
 
 # flake8: noqa
 
 logger = get_logger()
 
-SUBSET_LIST = [
-    'agronomy', 'anatomy', 'ancient_chinese', 'arts', 'astronomy', 'business_ethics', 'chinese_civil_service_exam',
-    'chinese_driving_rule', 'chinese_food_culture', 'chinese_foreign_policy', 'chinese_history', 'chinese_literature',
-    'chinese_teacher_qualification', 'college_actuarial_science', 'college_education', 'college_engineering_hydrology',
-    'college_law', 'college_mathematics', 'college_medical_statistics', 'clinical_knowledge', 'college_medicine',
-    'computer_science', 'computer_security', 'conceptual_physics', 'construction_project_management', 'economics',
-    'education', 'elementary_chinese', 'elementary_commonsense', 'elementary_information_and_technology',
-    'electrical_engineering', 'elementary_mathematics', 'ethnology', 'food_science', 'genetics', 'global_facts',
-    'high_school_biology', 'high_school_chemistry', 'high_school_geography', 'high_school_mathematics',
-    'high_school_physics', 'high_school_politics', 'human_sexuality', 'international_law', 'journalism',
-    'jurisprudence', 'legal_and_moral_basis', 'logical', 'machine_learning', 'management', 'marketing',
-    'marxist_theory', 'modern_chinese', 'nutrition', 'philosophy', 'professional_accounting', 'professional_law',
-    'professional_medicine', 'professional_psychology', 'public_relations', 'security_study', 'sociology',
-    'sports_science', 'traditional_chinese_medicine', 'virology', 'world_history', 'world_religions'
-]
-
 SUBJECT_MAPPING = {
     'agronomy': ['other', 'Other'],
     'anatomy': ['biology', 'STEM'],
@@ -102,112 +82,41 @@ SUBJECT_MAPPING = {
 }
 
 
-@Benchmark.register(
-    name='cmmlu',
-    pretty_name='C-MMLU',
-    tags=['Knowledge', 'MCQ', 'Chinese'],
-    description=
-    'C-MMLU is a benchmark designed to evaluate the performance of AI models on Chinese language tasks, including reading comprehension, text classification, and more.',
-    dataset_id='modelscope/cmmlu',
-    model_adapter=OutputType.GENERATION,
-    output_types=[OutputType.MULTIPLE_CHOICE, OutputType.GENERATION],
-    subset_list=SUBSET_LIST,
-    metric_list=['AverageAccuracy'],
-    few_shot_num=5,
-    train_split='dev',
-    eval_split='test',
-    prompt_template=
-    '以下是关于{subset_name}的单项选择题，请给出正确答案的选项。你的回答的最后一行应该是这样的格式：“答案：LETTER”（不带引号），其中 LETTER 是 A、B、C、D 中的一个。\n{query}',
+@register_benchmark(
+    BenchmarkMeta(
+        name='cmmlu',
+        pretty_name='C-MMLU',
+        tags=[Tags.KNOWLEDGE, Tags.MULTIPLE_CHOICE, Tags.CHINESE],
+        description=
+        'C-MMLU is a benchmark designed to evaluate the performance of AI models on Chinese language tasks, including reading comprehension, text classification, and more.',
+        dataset_id='evalscope/cmmlu',
+        metric_list=['acc'],
+        subset_list=list(SUBJECT_MAPPING.keys()),
+        few_shot_num=0,
+        train_split=None,
+        eval_split='test',
+        prompt_template=MultipleChoiceTemplate.CHINESE_SINGLE_ANSWER_TEMPLATE_COT,
+    )
 )
-class CMMLUAdapter(DataAdapter):
+class CMMLUAdapter(MultiChoiceAdapter):
 
     def __init__(self, **kwargs):
         super().__init__(**kwargs)
 
+        self.reformat_subset = True
         self.category_map = {k: v[-1] for k, v in SUBJECT_MAPPING.items()}
-        self.choices = ['A', 'B', 'C', 'D']
-
-    def load_from_disk(self, dataset_name_or_path, subset_list, work_dir, **kwargs) -> dict:
-        data_dict = defaultdict(dict)
-        for subset_name in subset_list:
-            for split_name in [self.train_split, self.eval_split]:
-                if os.path.exists(dataset_name_or_path):
-                    file_path = os.path.join(dataset_name_or_path, split_name, f'{subset_name}.csv')
-                else:
-                    file_path = os.path.join(work_dir, dataset_name_or_path, split_name, f'{subset_name}.csv')
-                if os.path.exists(file_path):
-                    data_dict[subset_name][split_name] = csv_to_list(file_path)
-
-        return data_dict
-
-    def gen_prompt(self, input_d: dict, subset_name: str, few_shot_list: list, **kwargs) -> dict:
-        """
-        Generate model prompt from raw input, unify the prompt format for CMMLU benchmark.
-
-        Args:
-            input_d (dict): The raw input. A single data format of the CMMLU:
-
-            {'Question': '下列关于重力的说法正确的是',
-            'A': '在地球周围的物体都要受到重力作用，与其运动状态无关',
-            'B': '对某一物体而言，重力的大小是一个恒量，不随物体的地理位置而改变',
-            'C': '重力就是地球对物体的吸引力，重力的方向总是竖直向下',
-            'D': '在地球表面各处的重力方向都是相同的',
-            'Answer': 'A'}
-
-        Returns:
-            {'data': [(context, continuation), ...]}
-
-        """
-        few_shot_prompts = [self._generate_prompt(input_d=sample, include_answer=True) for sample in few_shot_list]
-        context = '\n'.join(few_shot_prompts) + '\n'
-        context += self._generate_prompt(input_d=input_d, include_answer=False)
-
-        full_prompt = self.prompt_template.format(subset_name=self._format_subject(subset_name), query=context.strip())
-
-        return self.gen_prompt_data(full_prompt)
-
-    def get_gold_answer(self, input_d: dict) -> str:
-        # Get the gold choice
-        return input_d.get('Answer', '')
-
-    def parse_pred_result(self, result: str, raw_input_d: dict = None, eval_type: str = EvalType.CHECKPOINT) -> str:
-        """
-        Parse the model output to get the answer. Could be the best choice index.
-
-        Args:
-            result: Predicted answer from the model. Usually a string for chat.
-            raw_input_d: The raw input. Depending on the dataset.
-            eval_type: The evaluation type. 'checkpoint', 'service', 'custom'.
-
-        Returns:
-            The parsed answer. Depending on the dataset. Usually a string for chat.
-        """
-        if self.model_adapter == OutputType.MULTIPLE_CHOICE:
-            return result
-        else:
-            return ResponseParser.parse_first_option_with_choices(text=result, options=self.choices)
-
-    def match(self, gold: str, pred: str) -> float:
-        return exact_match(gold=gold, pred=pred)
-
-    def _generate_prompt(self, input_d: dict, include_answer=True) -> str:
-
-        input_choices: list = [input_d['A'], input_d['B'], input_d['C'], input_d['D']]
-
-        example: str = input_d['Question']
-        for j in range(len(self.choices)):
-            example += '\n{}. {}'.format(self.choices[j], input_choices[j])
 
-        example += '\nAnswer:'
-        if include_answer:
-            example += ' {}\n\n'.format(input_d['Answer'])
+    def record_to_sample(self, record) -> Sample:
 
-        return example
+        # choices: ["(A) 农业生产工具","(B) 土地","(C) 劳动力","(D) 资金"]
+        # remove the leading (A), (B), (C), (D)
+        raw_choices = record['choices']
+        choice_list = [choice[3:].strip() for choice in raw_choices]
 
-    @classmethod
-    def _format_subject(cls, subject):
-        l = subject.split('_')
-        s = ''
-        for entry in l:
-            s += ' ' + entry
-        return s
+        return Sample(
+            input=record['question'],
+            choices=choice_list,
+            target=record['answer'][1],  # answer is like "A"
+            subset_key=record['category'],
+            metadata={'subject': record['category']},
+        )

evalscope/benchmarks/competition_math/competition_math_adapter.py

@@ -1,125 +1,73 @@
 # Copyright (c) Alibaba, Inc. and its affiliates.
 # Copyright (c) EleutherAI, Inc. and its affiliates.
-import glob
-import json
-import os
-from collections import defaultdict
 
-from evalscope.benchmarks import Benchmark, DataAdapter
-from evalscope.metrics import extract_answer, math_equal, strip_answer_string
+from typing import Any, Dict
+
+from evalscope.api.benchmark import BenchmarkMeta, DefaultDataAdapter
+from evalscope.api.dataset import Sample
+from evalscope.api.evaluator import TaskState
+from evalscope.api.registry import register_benchmark
+from evalscope.constants import Tags
 from evalscope.utils.logger import get_logger
 
 # flake8: noqa
 
 logger = get_logger()
 
-
-@Benchmark.register(
-    name='competition_math',
-    pretty_name='MATH',
-    tags=['Mathematics'],
-    description=
-    'The MATH (Mathematics) benchmark is designed to evaluate the mathematical reasoning abilities of AI models through a variety of problem types, including arithmetic, algebra, geometry, and more.',
-    dataset_id='modelscope/competition_math',
-    subset_list=['Level 1', 'Level 2', 'Level 3', 'Level 4', 'Level 5'],
-    metric_list=['AveragePass@1'],
-    few_shot_num=4,
-    train_split=None,
-    eval_split='test',
-    prompt_template='{query}\nPlease reason step by step, and put your final answer within \\boxed{{}}.',
+PROMPT_TEMPLATE = """
+Problem:
+{question}
+
+Please reason step by step, and put your final answer within \\boxed{{}}.
+""".lstrip()
+
+FEWSHOT_TEMPLATE = """
+Here are some examples of how to solve similar problems:
+
+{fewshot}
+""".lstrip() + PROMPT_TEMPLATE
+
+
+@register_benchmark(
+    BenchmarkMeta(
+        name='competition_math',
+        pretty_name='MATH',
+        tags=[Tags.MATH, Tags.REASONING],
+        description=
+        'The MATH (Mathematics) benchmark is designed to evaluate the mathematical reasoning abilities of AI models through a variety of problem types, including arithmetic, algebra, geometry, and more.',
+        dataset_id='evalscope/competition_math',
+        subset_list=['Level 1', 'Level 2', 'Level 3', 'Level 4', 'Level 5'],
+        metric_list=[{
+            'acc': {
+                'numeric': True
+            }
+        }],
+        few_shot_num=4,
+        train_split='train',
+        eval_split='test',
+        prompt_template=PROMPT_TEMPLATE,
+        few_shot_prompt_template=FEWSHOT_TEMPLATE,
+    )
 )
-class CompetitionMathAdapter(DataAdapter):
-    """ To be tested for all models. """
+class CompetitionMathAdapter(DefaultDataAdapter):
 
     def __init__(self, **kwargs):
-
-        few_shot_num = kwargs.get('few_shot_num', 4)
-        if few_shot_num != 4 and few_shot_num != 0:
-            logger.error(f'The MATH benchmark ONLY supports 4-shot by system or 0-shot settings, '
-                         f'but got {few_shot_num}. Use 4-shot by default.')
-            kwargs['few_shot_num'] = 4
-
         super().__init__(**kwargs)
 
-    def load(self, **kwargs):
-        # default load all levels
-        kwargs['subset_list'] = ['default']
-        data_dict = super().load(**kwargs)
-        return self.reformat_subset(data_dict, subset_key='level')
-
-    def load_from_disk(self, dataset_name_or_path, subset_list, work_dir, **kwargs) -> dict:
-        data_dict = defaultdict(dict)
-        for subset_name in subset_list:
-            for split_name in [self.train_split, self.eval_split]:
-                if os.path.exists(dataset_name_or_path):
-                    split_dir = os.path.join(dataset_name_or_path, split_name)
-                else:
-                    split_dir = os.path.join(work_dir, dataset_name_or_path, split_name)
-                split_files = glob.glob(os.path.join(split_dir, '**', '*.json'))
-                split_data = []
-                for file_path in split_files:
-                    if os.path.exists(file_path):
-                        with open(file_path, 'r', encoding='utf-8') as f:
-                            split_data.append(json.load(f))
-                data_dict[subset_name][split_name] = split_data
-
-        return data_dict
-
-    def gen_prompt(self, input_d: dict, few_shot_list: list, **kwargs) -> dict:
-        """
-        Generate the prompt for the model input.
-
-        Args:
-            input_d: raw input dict.
-                {"problem": "How many vertical asymptotes does the graph of $y=\\frac{2}{x^2+x-6}$ have?", "level": "Level 3", "type": "Algebra", "solution": "The denominator of the rational function factors into $x^2+x-6=(x-2)(x+3)$. Since the numerator is always nonzero, there is a vertical asymptote whenever the denominator is $0$, which occurs for $x = 2$ and $x = -3$.  Therefore, the graph has $\\boxed{2}$ vertical asymptotes."}
-
-            few_shot_list:  few shot list. Each item is a raw input dict.
-            **kwargs:
-
-        Returns:
-            {'data': [prompt]}
-        """
-        use_fewshot = self.few_shot_num > 0
-        query = self._generate_prompt(input_d, use_fewshot=use_fewshot)
-        full_prompt = self.prompt_template.format(query=query)
-        return self.gen_prompt_data(full_prompt)
-
-    def get_gold_answer(self, input_d: dict) -> str:
-        # Extract the gold answer from the input dict.
-        return strip_answer_string(extract_answer(input_d['solution']))
-
-    def parse_pred_result(self, result: str, raw_input_d: dict = None, eval_type: str = 'checkpoint') -> str:
-        """
-        Parse the model output to get the answer. Could be the best choice index.
-
-        Args:
-            result: Predicted answer from the model. Usually a string for chat.
-            raw_input_d (dict): The raw input. Depending on the dataset.
-            eval_type: 'checkpoint' or 'service' or `custom`
-
-        Returns:
-            The parsed answer. Depending on the dataset. Usually a string for chat.
-        """
-        # Note: Use same extraction method for both of checkpoint/service/custom
-        result = strip_answer_string(extract_answer(result))
-        return result
+        self.reformat_subset = True
 
-    def match(self, gold: str, pred: str) -> float:
-        res = math_equal(pred, gold)
-        return 1.0 if res else 0.0
+    def record_to_sample(self, record: Dict[str, Any]) -> Sample:
+        from evalscope.metrics.math_parser import extract_answer
 
-    @classmethod
-    def _generate_prompt(cls, input_d: dict, use_fewshot: bool = True) -> str:
-        problem: str = input_d['problem']
+        return Sample(
+            input=record['problem'],
+            target=extract_answer(record['solution']),
+            subset_key=record['level'],
+            metadata={
+                'reasoning': record.get('solution', ''),
+                'type': record.get('type', ''),
+            },
+        )
 
-        if use_fewshot:
-            # Use 4-shot examples by system
-            context = (
-                'Problem:\nFind the domain of the expression $\\frac{{\sqrt{{x-2}}}}{{\sqrt{{5-x}}}}$.}}\nSolution:\nThe expressions inside each square root must be non-negative. Therefore, $x-2 \ge 0$, so $x\ge2$, and $5 - x \ge 0$, so $x \le 5$. Also, the denominator cannot be equal to zero, so $5-x>0$, which gives $x<5$. Therefore, the domain of the expression is $\\boxed{{[2,5)}}$.\nFinal Answer: The final answer is $[2,5)$. I hope it is correct.\n'
-                'Problem:\nIf $\det \mathbf{{A}} = 2$ and $\det \mathbf{{B}} = 12,$ then find $\det (\mathbf{{A}} \mathbf{{B}}).$\nSolution:\nWe have that $\det (\mathbf{{A}} \mathbf{{B}}) = (\det \mathbf{{A}})(\det \mathbf{{B}}) = (2)(12) = \\boxed{{24}}.$\nFinal Answer: The final answer is $24$. I hope it is correct.\n'
-                'Problem:\nTerrell usually lifts two 20-pound weights 12 times. If he uses two 15-pound weights instead, how many times must Terrell lift them in order to lift the same total weight?\nSolution:\nIf Terrell lifts two 20-pound weights 12 times, he lifts a total of $2\cdot 12\cdot20=480$ pounds of weight. If he lifts two 15-pound weights instead for $n$ times, he will lift a total of $2\cdot15\cdot n=30n$ pounds of weight. Equating this to 480 pounds, we can solve for $n$: \\begin{{align*}} 30n&=480\\\\ \Rightarrow\qquad n&=480/30=\\boxed{{16}} \end{{align*}}\nFinal Answer: The final answer is $16$. I hope it is correct.\n'
-                'Problem:\nIf the system of equations: \\begin{{align*}} 6x-4y&=a,\\\\ 6y-9x &=b. \end{{align*}}has a solution $(x, y)$ where $x$ and $y$ are both nonzero, find $\\frac{{a}}{{b}},$ assuming $b$ is nonzero.\nSolution:\nIf we multiply the first equation by $-\\frac{{3}}{{2}}$, we obtain $$6y-9x=-\\frac{{3}}{{2}}a.$$Since we also know that $6y-9x=b$, we have $$-\\frac{{3}}{{2}}a=b\Rightarrow\\frac{{a}}{{b}}=\\boxed{{-\\frac{{2}}{{3}}}}.$$\nFinal Answer: The final answer is $-\\frac{{2}}{{3}}$. I hope it is correct.\n'
-                f'Problem:\n{problem}\nSolution:\n')
-        else:
-            context = 'Problem:\n' + problem + '\nSolution:\n'
-        return context
+    def sample_to_fewshot(self, sample: Sample) -> str:
+        return f'Problem:\n{sample.input}\nSolution:\n{sample.target}'

evalscope/benchmarks/data_collection/data_collection_adapter.py

@@ -1,28 +1,35 @@
-import math
+import copy
 import os
-import re
-from typing import Any, Optional
+from collections import defaultdict
+from typing import Any, Dict, List
 
-from evalscope.benchmarks import Benchmark, DataAdapter
-from evalscope.constants import DEFAULT_DATASET_CACHE_DIR, EvalType, HubType
-from evalscope.utils.io_utils import jsonl_to_list
+from evalscope.api.benchmark import BenchmarkMeta, DataAdapter, DefaultDataAdapter
+from evalscope.api.dataset import DatasetDict, LocalDataLoader, Sample
+from evalscope.api.evaluator import TaskState
+from evalscope.api.metric import Score
+from evalscope.api.metric.scorer import AggScore, SampleScore
+from evalscope.api.model.model import Model
+from evalscope.api.registry import get_benchmark, register_benchmark
+from evalscope.config import TaskConfig
+from evalscope.constants import DataCollection, Tags
+from evalscope.report.generator import ReportGenerator
+from evalscope.report.report import Report
 from evalscope.utils.logger import get_logger
 
 logger = get_logger()
 
 
-@Benchmark.register(
-    name='data_collection',
-    dataset_id='',  # dataset_id need to be set
-    description='Data collection',
-    subset_list=['default'],
-    metric_list=['AverageAccuracy'],
-    few_shot_num=0,
-    train_split=None,
-    eval_split='test',
-    prompt_template='',
+@register_benchmark(
+    BenchmarkMeta(
+        name=DataCollection.NAME,
+        dataset_id='',  # dataset_id need to be set
+        description='Data collection',
+        metric_list=['acc'],
+        eval_split='test',
+        prompt_template='',
+    )
 )
-class DataCollectionAdapter(DataAdapter):
+class DataCollectionAdapter(DefaultDataAdapter):
 
     def __init__(self, **kwargs):
         """
@@ -30,43 +37,174 @@ class DataCollectionAdapter(DataAdapter):
         """
         super().__init__(**kwargs)
 
-    def load(self,
-             dataset_name_or_path: str = None,
-             subset_list: list = None,
-             work_dir: Optional[str] = DEFAULT_DATASET_CACHE_DIR,
-             datasets_hub: str = HubType.MODELSCOPE,
-             **kwargs) -> dict:
-        """
-        Load the dataset. Remote and local datasets are supported.
-        """
-        dataset_name_or_path = os.path.expanduser(dataset_name_or_path or self.dataset_id)
-        subset_list = subset_list or self.subset_list
-
+    def load(self):
         # Try to load dataset from local disk
+        dataset_name_or_path = self.dataset_id
         if os.path.exists(dataset_name_or_path):
             logger.info(f'Loading dataset from {dataset_name_or_path}')
-            dataset = jsonl_to_list(dataset_name_or_path)
-            if len(dataset) == 0:
-                raise ValueError(f'Local dataset is empty: {dataset_name_or_path}')
+            dataset_path = dataset_name_or_path
         else:
             from modelscope import dataset_snapshot_download
 
             # Load dataset from remote
-            logger.info(f'Loading dataset from {datasets_hub}: > dataset_name: {dataset_name_or_path}')
+            logger.info(f'Loading dataset from modelscope: > dataset_name: {dataset_name_or_path}')
+            # download dataset snapshot
+            dataset_path = dataset_snapshot_download(dataset_name_or_path, allow_file_pattern='*.jsonl')
+
+        dataset = LocalDataLoader(
+            data_id_or_path=dataset_path,
+            split=self.eval_split,
+            sample_fields=self.record_to_sample,
+            subset=self.default_subset,
+            limit=self.limit,
+            repeats=self.repeats
+        ).load()
+
+        test_dataset = DatasetDict({self.default_subset: dataset})
+
+        return test_dataset, None
+
+    def record_to_sample(self, record: Dict[str, Any]) -> Sample:
+        """
+        Convert a data record to a Sample object. Every record is a DatasetEntry.
+
+        Args:
+            record (Dict[str, Any]): Input data record.
+
+        Returns:
+            Sample: Sample object with input, target, and metadata.
+        """
+        from evalscope.collections import DatasetEntry
+
+        entry = DatasetEntry.model_validate(record)
+        sample = Sample.model_validate(entry.prompt)
+
+        record_without_prompt = copy.deepcopy(record)
+        del record_without_prompt['prompt']
+        sample.metadata[DataCollection.INFO] = record_without_prompt  # keep all metadata
+        return sample
+
+    def _post_process_samples(self):
+        """Post process of each sample"""
+        self._initialize_adapters()
+
+    def _initialize_adapters(self):
+        """Init adapters for each dataset and create dataset id map"""
+        self.dataset_adapters: Dict[str, DataAdapter] = {}
+        self.dataset_name_map = defaultdict(lambda: defaultdict(list))
+
+        # load dataset args
+        dataset_args = copy.deepcopy(self._task_config.dataset_args)
+        common_args = dataset_args.get(DataCollection.NAME, {})
+
+        # Iterate through each sample in the dataset
+        dataset = self.test_dataset[self.default_subset]
+        for sample in dataset:
+            collection_info = sample.metadata.get(DataCollection.INFO, {})
+            dataset_name = collection_info.get('dataset_name', '')
+            subset_name = collection_info.get('subset_name', '')
+            # create id mapping
+            self.dataset_name_map[dataset_name][subset_name].append(sample.id)
+
+            # update dataset args
+            cur_dataset_args = dataset_args.get(dataset_name, {})
+            cur_dataset_args.update(common_args)
+
+            # Initialize dataset adapter
+            if dataset_name not in self.dataset_adapters:
+                config = TaskConfig(dataset_args={dataset_name: cur_dataset_args})
+                self.dataset_adapters[dataset_name] = get_benchmark(dataset_name, config=config)
+
+    def _get_adapter(self, metadata: Dict[str, Any]) -> DataAdapter:
+        collection_info = metadata.get(DataCollection.INFO, {})
+        dataset_name = collection_info.get('dataset_name', '')
+        return self.dataset_adapters.get(dataset_name)
+
+    def run_inference(self, model, sample, output_dir, **kwargs) -> TaskState:
+        data_adapter = self._get_adapter(sample.metadata)
+        if not data_adapter:
+            raise ValueError(f'No data adapter found for sample: {sample}')
+
+        return data_adapter.run_inference(model, sample, output_dir, **kwargs)
+
+    def calculate_metrics(self, task_state) -> SampleScore:
+        data_adapter = self._get_adapter(task_state.metadata)
+        if not data_adapter:
+            raise ValueError(f'No data adapter found for task state: {task_state}')
+
+        return data_adapter.calculate_metrics(task_state)
+
+    def aggregate_scores(self, sample_scores: List[SampleScore]):
+        import pandas as pd
+        from tabulate import tabulate
+
+        data = []
+        for sample_score in sample_scores:
+            collection_info = sample_score.sample_metadata[DataCollection.INFO]
+            for metric_name, value in sample_score.score.value.items():
+                data.append(
+                    dict(
+                        task_type=collection_info['task_type'],
+                        categories=tuple(collection_info['categories']),
+                        dataset_name=collection_info['dataset_name'],
+                        subset_name=collection_info['subset_name'],
+                        tags=collection_info['tags'],
+                        sample_id=sample_score.sample_id,
+                        metric=metric_name,
+                        score=value
+                    )
+                )
+
+        df = pd.DataFrame(data)
+
+        def aggregate_and_sort(df, group_by_cols):
+            # aggregate by group_by_cols, and calculate average_score and count
+            report_df = df.groupby(group_by_cols) \
+                .agg(average_score=('score', 'mean'), count=('score', 'size')) \
+                .reset_index()
+            report_df['average_score'] = report_df['average_score'].round(4)
+            report_df = report_df.sort_values(by='count', ascending=False) \
+                .to_dict(orient='records')
+            return report_df
+
+        # multi-level aggregation
+        subset_report_df = aggregate_and_sort(df, ['task_type', 'metric', 'dataset_name', 'subset_name'])
+        dataset_report_df = aggregate_and_sort(df, ['task_type', 'metric', 'dataset_name'])
+        task_report_df = aggregate_and_sort(df, ['task_type', 'metric'])
+
+        # explode tags to multiple rows
+        df_exploded_tags = df.explode('tags')
+        tag_report_df = aggregate_and_sort(df_exploded_tags, ['tags', 'metric'])
 
-            dataset_path = dataset_snapshot_download(
-                dataset_name_or_path, cache_dir=work_dir, allow_file_pattern='*.jsonl')
-            # find the jsonl file
-            dataset_files = [os.path.join(dataset_path, f) for f in os.listdir(dataset_path) if f.endswith('.jsonl')]
-            dataset = jsonl_to_list(dataset_files[0])
+        # process multi-level categories
+        df_categories = df.copy()
+        # multi-level aggregation for categories
+        max_depth = df_categories['categories'].apply(len).max()
+        for level in range(max_depth):
+            df_categories[f'category{level}'] = df_categories['categories'].apply(
+                lambda x: x[level] if len(x) > level else ''
+            )
+        category_report_df = aggregate_and_sort(
+            df_categories, [f'category{level}' for level in range(max_depth)] + ['metric']
+        )
 
-        return dataset
+        # convert to dict format
+        report_dict = {
+            'subset_level': subset_report_df,
+            'dataset_level': dataset_report_df,
+            'task_level': task_report_df,
+            'tag_level': tag_report_df,
+            'category_level': category_report_df,
+        }
 
-    def get_gold_answer(self, input_d: Any) -> Any:
-        return super().get_gold_answer(input_d)
+        # record report
+        for level, data in report_dict.items():
+            table = tabulate(data, headers='keys', tablefmt='pretty', showindex=False)
+            logger.info(f'{level} Report:\n{table}')
 
-    def match(self, gold: Any, pred: Any) -> Any:
-        return super().match(gold, pred)
+        return df
 
-    def parse_pred_result(self, result: Any, raw_input_d: dict = None, eval_type: str = EvalType.CHECKPOINT) -> Any:
-        return super().parse_pred_result(result, raw_input_d, eval_type)
+    def generate_report(self, scores, model_name, output_dir, **kwargs) -> Report:
+        df = scores[self.default_subset]
+        report = ReportGenerator.gen_collection_report(df, self.name, model_name)
+        return report