evalscope 0.17.1__py3-none-any.whl → 1.0.1__py3-none-any.whl

evalscope/perf/benchmark.py

@@ -42,6 +42,8 @@ async def get_requests(args: Arguments, api_plugin: 'ApiPluginBase') -> AsyncGen
         try:
             for messages in message_generator.build_messages():
                 dataset_messages.append(messages)
+                if len(dataset_messages) >= args.number:
+                    break
         except StopIteration:
             pass
 
@@ -189,7 +191,8 @@ async def benchmark(args: Arguments) -> Tuple[Dict, Dict]:
     await connect_test(args, api_plugin)
     # start statistic benchmark metric
     statistic_benchmark_metric_task = asyncio.create_task(
-        statistic_benchmark_metric(benchmark_data_queue, args, api_plugin))
+        statistic_benchmark_metric(benchmark_data_queue, args, api_plugin)
+    )
     # start send request
     semaphore = asyncio.Semaphore(args.parallel)
     send_request_tasks: List[asyncio.Task] = []

evalscope/perf/http_client.py

@@ -26,7 +26,8 @@ class AioHttpClient:
         self.api_plugin = api_plugin
         self.client = aiohttp.ClientSession(
             timeout=aiohttp.ClientTimeout(connect=self.connect_timeout, sock_read=self.read_timeout),
-            trace_configs=[self._create_trace_config()] if args.debug else [])
+            trace_configs=[self._create_trace_config()] if args.debug else []
+        )
 
     async def __aenter__(self):
         pass
@@ -105,7 +106,8 @@ async def test_connection(args: Arguments, api_plugin: 'ApiPluginBase') -> bool:
     while True:
         try:
             is_error, state_code, response_data = await asyncio.wait_for(
-                attempt_connection(), timeout=args.connect_timeout)
+                attempt_connection(), timeout=args.connect_timeout
+            )
             if not is_error:
                 logger.info('Test connection successful.')
                 return True

evalscope/perf/plugin/api/custom_api.py

@@ -153,7 +153,8 @@ class CustomPlugin(ApiPluginBase):
 
             # If no usage information and no tokenizer, raise an error
             raise ValueError(
-                'Cannot determine token counts: no usage information in response and no tokenizer provided.')
+                'Cannot determine token counts: no usage information in response and no tokenizer provided.'
+            )
 
         except Exception as e:
             logger.error(f'Error parsing responses: {e}')
@@ -186,8 +187,7 @@ class CustomPlugin(ApiPluginBase):
             data = json.dumps(body, ensure_ascii=False)
 
             # Send the request
-            async with client_session.request(
-                'POST', url=url, data=data, headers=headers) as response:  # noqa: E125
+            async with client_session.request('POST', url=url, data=data, headers=headers) as response:  # noqa: E125
                 # Get the status code
                 status_code = response.status
 
@@ -244,6 +244,7 @@ if __name__ == '__main__':
         api='custom',  # Use the custom API plugin registered above
         dataset='openqa',
         number=1,
-        max_tokens=10)
+        max_tokens=10
+    )
 
     run_perf_benchmark(args)

evalscope/perf/plugin/api/openai_api.py

@@ -159,13 +159,15 @@ class OpenaiPlugin(DefaultApiPlugin):
                 input_tokens += len(self.tokenizer.encode(request['messages'][0]['content']))
                 output_tokens += len(self.tokenizer.encode(full_response_content))
         else:
-            raise ValueError('Error: Unable to retrieve usage information\n\n'
-                             'This error occurs when:\n'
-                             '1. The API response does not contain usage data, AND\n'
-                             '2. No tokenizer has been specified or found.\n\n'
-                             'To resolve this issue, do ONE of the following:\n'
-                             "a) Ensure that the API you're using supports and returns usage information, OR\n"
-                             'b) Specify a tokenizer using the `--tokenizer-path` parameter.\n\n'
-                             'If you continue to experience issues, '
-                             'please open an issue on our GitHub repository https://github.com/modelscope/evalscope .')
+            raise ValueError(
+                'Error: Unable to retrieve usage information\n\n'
+                'This error occurs when:\n'
+                '1. The API response does not contain usage data, AND\n'
+                '2. No tokenizer has been specified or found.\n\n'
+                'To resolve this issue, do ONE of the following:\n'
+                "a) Ensure that the API you're using supports and returns usage information, OR\n"
+                'b) Specify a tokenizer using the `--tokenizer-path` parameter.\n\n'
+                'If you continue to experience issues, '
+                'please open an issue on our GitHub repository https://github.com/modelscope/evalscope .'
+            )
         return input_tokens, output_tokens

evalscope/perf/plugin/datasets/custom.py

@@ -17,7 +17,8 @@ class CustomDatasetPlugin(DatasetPluginBase):
         for item in self.dataset_line_by_line(self.query_parameters.dataset_path):
             prompt = item.strip()
             if len(prompt) > self.query_parameters.min_prompt_length and len(
-                    prompt) < self.query_parameters.max_prompt_length:
+                prompt
+            ) < self.query_parameters.max_prompt_length:
                 if self.query_parameters.apply_chat_template:
                     message = self.create_message(prompt)
                     yield [message]

evalscope/perf/plugin/datasets/flickr8k.py

@@ -24,5 +24,5 @@ class FlickrDatasetPlugin(DatasetPluginBase):
             text = item['txt']
             base64_image = PIL_to_base64(pil_image)
 
-            message = self.create_message(text=text, image_url=f'data:image/jpeg;base64,{base64_image}')
+            message = self.create_message(text=text, image_urls=f'data:image/jpeg;base64,{base64_image}')
             yield [message]

evalscope/perf/plugin/datasets/kontext_bench.py

@@ -24,5 +24,5 @@ class KontextDatasetPlugin(DatasetPluginBase):
             text = item['instruction']
             base64_image = PIL_to_base64(pil_image)
 
-            message = self.create_message(text=text, image_url=f'data:image/jpeg;base64,{base64_image}')
+            message = self.create_message(text=text, image_urls=f'data:image/jpeg;base64,{base64_image}')
             yield [message]

evalscope/perf/plugin/datasets/line_by_line.py

@@ -18,7 +18,8 @@ class LineByLineDatasetPlugin(DatasetPluginBase):
         for item in self.dataset_line_by_line(self.query_parameters.dataset_path):
             prompt = item.strip()
             if len(prompt) > self.query_parameters.min_prompt_length and len(
-                    prompt) < self.query_parameters.max_prompt_length:
+                prompt
+            ) < self.query_parameters.max_prompt_length:
                 if self.query_parameters.apply_chat_template:
                     message = self.create_message(prompt)
                     yield [message]

evalscope/perf/plugin/datasets/longalpaca.py

@@ -23,7 +23,8 @@ class LongAlpacaDatasetPlugin(DatasetPluginBase):
         for item in ds:
             prompt = item['instruction'].strip()
             if len(prompt) > self.query_parameters.min_prompt_length and len(
-                    prompt) < self.query_parameters.max_prompt_length:
+                prompt
+            ) < self.query_parameters.max_prompt_length:
                 if self.query_parameters.apply_chat_template:
                     message = self.create_message(prompt)
                     yield [message]

evalscope/perf/plugin/datasets/openqa.py

@@ -27,8 +27,10 @@ class OpenqaDatasetPlugin(DatasetPluginBase):
         for item in self.dataset_line_by_line(self.query_parameters.dataset_path):
             item = json.loads(item)
             prompt = item['question'].strip()
-            if (len(prompt) > self.query_parameters.min_prompt_length
-                    and len(prompt) < self.query_parameters.max_prompt_length):
+            if (
+                len(prompt) > self.query_parameters.min_prompt_length
+                and len(prompt) < self.query_parameters.max_prompt_length
+            ):
                 if self.query_parameters.apply_chat_template:
                     message = self.create_message(prompt)
                     yield [message]

evalscope/perf/utils/benchmark_util.py

@@ -1,8 +1,8 @@
 import time
-import torch
 from dataclasses import dataclass, field
 from typing import Any, List, Optional, Tuple
 
+from evalscope.utils.import_utils import check_import
 from evalscope.utils.logger import get_logger
 
 logger = get_logger()
@@ -33,8 +33,8 @@ class BenchmarkData:
         if len(self.chunk_times) > 1:
             self.first_chunk_latency = self.chunk_times[0] - self.start_time
             # remove the first chunk time from the total latency
-            self.time_per_output_token = (self.query_latency - self.first_chunk_latency) / (
-                self.completion_tokens - 1) if self.completion_tokens > 1 else 0.0
+            self.time_per_output_token = (self.query_latency - self.first_chunk_latency
+                                          ) / (self.completion_tokens - 1) if self.completion_tokens > 1 else 0.0
             self.inter_chunk_latency = [t2 - t1 for t1, t2 in zip(self.chunk_times[:-1], self.chunk_times[1:])]
         else:
             self.first_chunk_latency = self.query_latency
@@ -44,10 +44,13 @@ class BenchmarkData:
             api_plugin.parse_responses(self.response_messages, request=self.request)
 
     def update_gpu_usage(self):
-        total_memory = 0
-        for i in range(torch.cuda.device_count()):
-            total_memory += (torch.cuda.max_memory_allocated(i) / 2**30)  # GB
-        self.max_gpu_memory_cost = max(self.max_gpu_memory_cost, total_memory)
+        if check_import('torch'):
+
+            import torch
+            total_memory = 0
+            for i in range(torch.cuda.device_count()):
+                total_memory += (torch.cuda.max_memory_allocated(i) / 2**30)  # GB
+            self.max_gpu_memory_cost = max(self.max_gpu_memory_cost, total_memory)
 
 
 class Metrics:
@@ -126,11 +129,13 @@ class BenchmarkMetrics:
             self.avg_completion_tokens = self.n_total_completion_tokens / self.n_succeed_queries
             self.avg_input_token_per_seconds = self.n_total_prompt_tokens / self.total_first_chunk_latency
             self.avg_output_token_per_seconds = self.n_total_completion_tokens / self.total_time
-            self.avg_total_token_per_seconds = (self.n_total_prompt_tokens
-                                                + self.n_total_completion_tokens) / self.total_time
+            self.avg_total_token_per_seconds = (
+                self.n_total_prompt_tokens + self.n_total_completion_tokens
+            ) / self.total_time
             self.avg_time_per_token = self.n_time_per_output_token / self.n_succeed_queries
             self.avg_inter_token_latency = sum(self.n_total_inter_token_latency) / len(
-                self.n_total_inter_token_latency) if self.n_total_inter_token_latency else 0.0
+                self.n_total_inter_token_latency
+            ) if self.n_total_inter_token_latency else 0.0
             self.qps = self.n_succeed_queries / self.total_time
         except ZeroDivisionError as e:
             logger.exception(e)

evalscope/perf/utils/db_util.py

@@ -56,7 +56,8 @@ def transpose_results(data):
 
 
 def create_result_table(cursor):
-    cursor.execute(f'''CREATE TABLE IF NOT EXISTS result(
+    cursor.execute(
+        f'''CREATE TABLE IF NOT EXISTS result(
                       {DatabaseColumns.REQUEST} TEXT,
                       {DatabaseColumns.START_TIME} REAL,
                       {DatabaseColumns.CHUNK_TIMES} TEXT,
@@ -69,7 +70,8 @@ def create_result_table(cursor):
                       {DatabaseColumns.COMPLETION_TOKENS} INTEGER,
                       {DatabaseColumns.MAX_GPU_MEMORY_COST} REAL,
                       {DatabaseColumns.TIME_PER_OUTPUT_TOKEN} REAL
-                   )''')
+                   )'''
+    )
 
 
 def insert_benchmark_data(cursor: sqlite3.Cursor, benchmark_data: BenchmarkData):
@@ -89,9 +91,10 @@ def insert_benchmark_data(cursor: sqlite3.Cursor, benchmark_data: BenchmarkData)
 
     if benchmark_data.success:
         # Add additional columns for success case
-        additional_columns = (benchmark_data.query_latency, benchmark_data.first_chunk_latency,
-                              benchmark_data.prompt_tokens, benchmark_data.completion_tokens,
-                              benchmark_data.max_gpu_memory_cost, benchmark_data.time_per_output_token)
+        additional_columns = (
+            benchmark_data.query_latency, benchmark_data.first_chunk_latency, benchmark_data.prompt_tokens,
+            benchmark_data.completion_tokens, benchmark_data.max_gpu_memory_cost, benchmark_data.time_per_output_token
+        )
         query = f"""INSERT INTO result(
                       {DatabaseColumns.REQUEST}, {DatabaseColumns.START_TIME}, {DatabaseColumns.CHUNK_TIMES},
                       {DatabaseColumns.SUCCESS}, {DatabaseColumns.RESPONSE_MESSAGES}, {DatabaseColumns.COMPLETED_TIME},
@@ -124,7 +127,7 @@ def get_result_db_path(args: Arguments):
 
     logger.info(f'Save the data base to: {result_db_path}')
     if os.path.exists(result_db_path):
-        logger.warning('The db file exists, delete it and start again!.')
+        logger.error(f'The db file {result_db_path} exists, delete it and start again!.')
         sys.exit(1)
 
     return result_db_path

evalscope/perf/utils/local_server.py

@@ -1,6 +1,5 @@
 import os
 import subprocess
-import torch
 import uvicorn
 from contextlib import asynccontextmanager
 from dataclasses import dataclass
@@ -10,6 +9,7 @@ from sse_starlette.sse import EventSourceResponse
 
 from evalscope.perf.arguments import Arguments
 from evalscope.utils.chat_service import ChatCompletionRequest, ChatService, ModelList, TextCompletionRequest
+from evalscope.utils.import_utils import check_import
 from evalscope.utils.logger import get_logger
 
 logger = get_logger()
@@ -61,8 +61,12 @@ class ServerSentEvent(object):
 @asynccontextmanager
 async def lifespan(app: FastAPI):
     yield
-    if torch.cuda.is_available():
-        torch.cuda.empty_cache()
+    try:
+        import torch
+        if torch.cuda.is_available():
+            torch.cuda.empty_cache()
+    except ImportError:
+        pass
 
 
 def create_app(model, attn_implementation=None) -> FastAPI:
@@ -98,10 +102,14 @@ def create_app(model, attn_implementation=None) -> FastAPI:
 def start_app(args: Arguments):
     logger.info('Starting local server, please wait...')
     if args.api == 'local':
+        check_import('torch', 'torch', raise_error=True)
+
         app = create_app(args.model, args.attn_implementation)
         uvicorn.run(app, host='0.0.0.0', port=args.port, workers=1)
 
     elif args.api == 'local_vllm':
+        import torch
+
         os.environ['VLLM_USE_MODELSCOPE'] = 'True'
         os.environ['VLLM_ALLOW_LONG_MAX_MODEL_LEN'] = '1'
         os.environ['VLLM_WORKER_MULTIPROC_METHOD'] = 'spawn'

evalscope/perf/utils/rich_display.py

@@ -32,8 +32,9 @@ def analyze_results(all_results):
             avg_tps = total_metrics.get(Metrics.OUTPUT_TOKEN_THROUGHPUT, 0)
             avg_ttft = total_metrics.get(Metrics.AVERAGE_TIME_TO_FIRST_TOKEN, 0)
             p99_ttft = percentile_metrics.get(PercentileMetrics.TTFT)[percentiles.index('99%')]
-            success_rate = (total_metrics.get(Metrics.SUCCEED_REQUESTS, 0)
-                            / total_metrics.get(Metrics.TOTAL_REQUESTS, 1)) * 100
+            success_rate = (
+                total_metrics.get(Metrics.SUCCEED_REQUESTS, 0) / total_metrics.get(Metrics.TOTAL_REQUESTS, 1)
+            ) * 100
             avg_tpot = total_metrics.get(Metrics.AVERAGE_TIME_PER_OUTPUT_TOKEN, 0)
             p99_tpot = percentile_metrics.get(PercentileMetrics.TPOT)[percentiles.index('99%')]
 
@@ -55,12 +56,13 @@ def analyze_results(all_results):
                 f'{p99_tpot:.3f}' if p99_tpot is not None else 'N/A',
             ])
 
-            total_tokens += total_metrics.get(Metrics.AVERAGE_OUTPUT_TOKENS_PER_REQUEST, 0) * total_metrics.get(
-                Metrics.SUCCEED_REQUESTS, 0)
+            total_tokens += total_metrics.get(Metrics.AVERAGE_OUTPUT_TOKENS_PER_REQUEST,
+                                              0) * total_metrics.get(Metrics.SUCCEED_REQUESTS, 0)
             total_time += total_metrics.get(Metrics.TIME_TAKEN_FOR_TESTS, 0)
         except Exception as e:
             logger.warning(
-                f"Warning: Error processing results for concurrency {result.get('concurrency', 'unknown')}: {str(e)}")
+                f"Warning: Error processing results for concurrency {result.get('concurrency', 'unknown')}: {str(e)}"
+            )
             continue
 
     if not summary:
@@ -138,7 +140,8 @@ def print_summary(all_results, model_name):
                 f'{float(row[8]):.3f}',  # Average TPOT
                 f'{float(row[9]):.3f}',  # P99 TPOT
                 row[6],  # Success Rate
-                style=row_style)
+                style=row_style
+            )
         except ValueError as e:
             console.print(f'Warning: Error processing row data: {str(e)}', style='bold red')
             continue
@@ -156,8 +159,9 @@ def print_summary(all_results, model_name):
         perf_info.add_column('Value', style='green', width=40)
 
         perf_info.add_row('Highest RPS', f'Concurrency {summary[best_rps_idx][0]} ({summary[best_rps_idx][1]} req/sec)')
-        perf_info.add_row('Lowest Latency',
-                          f'Concurrency {summary[best_latency_idx][0]} ({summary[best_latency_idx][2]} seconds)')
+        perf_info.add_row(
+            'Lowest Latency', f'Concurrency {summary[best_latency_idx][0]} ({summary[best_latency_idx][2]} seconds)'
+        )
 
         console.print('\n')
         console.print(perf_info)
@@ -166,7 +170,8 @@ def print_summary(all_results, model_name):
         recommendations = []
         if best_rps_idx == len(summary) - 1:
             recommendations.append(
-                'The system seems not to have reached its performance bottleneck, try higher concurrency')
+                'The system seems not to have reached its performance bottleneck, try higher concurrency'
+            )
         elif best_rps_idx == 0:
             recommendations.append('Consider lowering concurrency, current load may be too high')
         else:
@@ -175,7 +180,8 @@ def print_summary(all_results, model_name):
         success_rate = float(summary[-1][6][:-1])
         if success_rate < 95:
             recommendations.append(
-                'Success rate is low at high concurrency, check system resources or reduce concurrency')
+                'Success rate is low at high concurrency, check system resources or reduce concurrency'
+            )
 
         recommend_text = Text('\nPerformance Recommendations:', style='bold cyan')
         console.print(recommend_text)

evalscope/report/__init__.py

@@ -6,7 +6,7 @@ from evalscope.utils.import_utils import _LazyModule
 if TYPE_CHECKING:
     from .combinator import gen_table, get_data_frame, get_report_list
     from .generator import ReportGenerator
-    from .utils import Category, Report, ReportKey, Subset
+    from .report import Category, Report, ReportKey, Subset
 
 else:
     _import_structure = {
@@ -14,12 +14,11 @@ else:
             'gen_table',
             'get_data_frame',
             'get_report_list',
-            'gen_report_table',
         ],
         'generator': [
             'ReportGenerator',
         ],
-        'utils': [
+        'report': [
             'Category',
             'Report',
             'ReportKey',

evalscope/report/combinator.py

@@ -6,7 +6,7 @@ import pandas as pd
 from tabulate import tabulate
 from typing import List, Tuple
 
-from evalscope.report.utils import Report
+from evalscope.report.report import Report
 from evalscope.utils.logger import get_logger
 
 logger = get_logger()
@@ -32,25 +32,30 @@ def get_report_list(reports_path_list: List[str]) -> List[Report]:
     return report_list
 
 
-def get_data_frame(report_list: List[Report],
-                   flatten_metrics: bool = True,
-                   flatten_categories: bool = True,
-                   add_overall_metric: bool = False) -> pd.DataFrame:
+def get_data_frame(
+    report_list: List[Report],
+    flatten_metrics: bool = True,
+    flatten_categories: bool = True,
+    add_overall_metric: bool = False
+) -> pd.DataFrame:
     tables = []
     for report in report_list:
         df = report.to_dataframe(
             flatten_metrics=flatten_metrics,
             flatten_categories=flatten_categories,
-            add_overall_metric=add_overall_metric)
+            add_overall_metric=add_overall_metric
+        )
         tables.append(df)
     return pd.concat(tables, ignore_index=True)
 
 
-def gen_table(reports_path_list: list[str] = None,
-              report_list: list[Report] = None,
-              flatten_metrics: bool = True,
-              flatten_categories: bool = True,
-              add_overall_metric: bool = False) -> str:
+def gen_table(
+    reports_path_list: list[str] = None,
+    report_list: list[Report] = None,
+    flatten_metrics: bool = True,
+    flatten_categories: bool = True,
+    add_overall_metric: bool = False
+) -> str:
     """
     Generates a formatted table from a list of report paths or Report objects.
 
@@ -78,7 +83,8 @@ def gen_table(reports_path_list: list[str] = None,
         report_list,
         flatten_metrics=flatten_metrics,
         flatten_categories=flatten_categories,
-        add_overall_metric=add_overall_metric)
+        add_overall_metric=add_overall_metric
+    )
     return tabulate(table, headers=table.columns, tablefmt='grid', showindex=False)
 
 

evalscope/report/generator.py

@@ -3,29 +3,54 @@ from pandas import DataFrame
 from typing import TYPE_CHECKING
 
 from evalscope.constants import DataCollection
-from evalscope.report.utils import *
+from evalscope.report.report import *
 
 if TYPE_CHECKING:
-    from evalscope.benchmarks import DataAdapter
+    from evalscope.api.benchmark import DataAdapter
+    from evalscope.api.metric import AggScore
 
 
 class ReportGenerator:
 
     @staticmethod
-    def gen_report(subset_score_map: dict, model_name: str, data_adapter: 'DataAdapter', **kwargs) -> Report:
+    def gen_collection_report(df: DataFrame, all_dataset_name: str, model_name: str) -> Report:
+        metrics_list = []
+        for metric_name, group_metric in df.groupby('metric', sort=False):
+            categories = []
+            for category_name, group_category in group_metric.groupby('categories'):
+                subsets = []
+                for (dataset_name, subset_name), group_subset in group_category.groupby(['dataset_name',
+                                                                                         'subset_name']):
+                    avg_score = group_subset['score'].mean()
+                    num = group_subset['score'].count()
+                    subsets.append(Subset(name=f'{dataset_name}/{subset_name}', score=float(avg_score), num=int(num)))
+                categories.append(Category(name=category_name, subsets=subsets))
+            metrics_list.append(Metric(name=metric_name, categories=categories))
+        return Report(
+            name=DataCollection.NAME, metrics=metrics_list, dataset_name=all_dataset_name, model_name=model_name
+        )
+
+    @staticmethod
+    def generate_report(
+        score_dict: Dict[str, List['AggScore']],
+        model_name: str,
+        data_adapter: 'DataAdapter',
+        add_aggregation_name: bool = True
+    ) -> Report:
         """
         Generate a report for a specific dataset based on provided subset scores.
 
         Args:
             subset_score_map (dict): A mapping from subset names to a list of score dictionaries.
-                    {
-                        'subset_name': [
-                            {'metric_name': 'AverageAccuracy', 'score': 0.3389, 'num': 100},
-                            {'metric_name': 'WeightedAverageAccuracy', 'score': 0.3389, 'num': 100}
-                        ],
-                        ...
-                    }
-            report_name (str): The name of the report to generate.
+            ```
+            {
+                'subset_name': [
+                    AggScore={'metric_name': 'AverageAccuracy', 'score': 0.3389, 'num': 100},
+                    AggScore={'metric_name': 'WeightedAverageAccuracy', 'score': 0.3389, 'num': 100}
+                ],
+                ...
+            }
+            ```
             data_adapter (DataAdapter): An adapter object for data handling.
 
         Returns:
@@ -48,18 +73,25 @@ class ReportGenerator:
             1  ARC-Challenge    0.5    2    [default]  AverageAccuracy
             """
             subsets = []
-            for subset_name, scores in subset_score_map.items():
-                for score_item in scores:
+            for subset_name, agg_scores in score_dict.items():
+                for agg_score_item in agg_scores:
                     categories = category_map.get(subset_name, ['default'])
+                    if add_aggregation_name and agg_score_item.aggregation_name:
+                        metric_name = f'{agg_score_item.aggregation_name}_{agg_score_item.metric_name}'
+                    else:
+                        metric_name = agg_score_item.metric_name
+
                     if isinstance(categories, str):
                         categories = [categories]
                     subsets.append(
                         dict(
                             name=subset_name,
-                            score=score_item['score'],
-                            num=score_item['num'],
-                            metric_name=score_item['metric_name'],
-                            categories=tuple(categories)))
+                            score=agg_score_item.score,
+                            num=agg_score_item.num,
+                            metric_name=metric_name,
+                            categories=tuple(categories)
+                        )
+                    )
             df = pd.DataFrame(subsets)
             return df
 
@@ -83,22 +115,6 @@ class ReportGenerator:
             dataset_name=dataset_name,
             model_name=model_name,
             dataset_description=data_adapter.description,
-            dataset_pretty_name=data_adapter.pretty_name)
+            dataset_pretty_name=data_adapter.pretty_name
+        )
         return report
-
-    @staticmethod
-    def gen_collection_report(df: DataFrame, all_dataset_name: str, model_name: str) -> Report:
-        categories = []
-        for category_name, group_category in df.groupby('categories'):
-            subsets = []
-            for (dataset_name, subset_name), group_subset in group_category.groupby(['dataset_name', 'subset_name']):
-                avg_score = group_subset['score'].mean()
-                num = group_subset['score'].count()
-                subsets.append(Subset(name=f'{dataset_name}/{subset_name}', score=float(avg_score), num=int(num)))
-
-            categories.append(Category(name=category_name, subsets=subsets))
-        return Report(
-            name=DataCollection.NAME,
-            metrics=[Metric(name='Average', categories=categories)],
-            dataset_name=all_dataset_name,
-            model_name=model_name)

evalscope/report/{utils.py → report.py}

@@ -152,10 +152,12 @@ class Report:
             data = json.load(f)
         return cls.from_dict(data)
 
-    def to_dataframe(self,
-                     flatten_metrics: bool = True,
-                     flatten_categories: bool = True,
-                     add_overall_metric: bool = False) -> pd.DataFrame:
+    def to_dataframe(
+        self,
+        flatten_metrics: bool = True,
+        flatten_categories: bool = True,
+        add_overall_metric: bool = False
+    ) -> pd.DataFrame:
         """
         Convert the report to a pandas DataFrame.
         Args:
@@ -201,8 +203,8 @@ class Report:
         # multi-level aggregation for categories
         max_depth = df_categories[ReportKey.category_name].apply(len).max()
         for level in range(max_depth):
-            df_categories[f'{ReportKey.category_prefix}{level}'] = df_categories[ReportKey.category_name].apply(
-                lambda x: x[level] if len(x) > level else None)
+            df_categories[f'{ReportKey.category_prefix}{level}'] = df_categories[
+                ReportKey.category_name].apply(lambda x: x[level] if len(x) > level else None)
 
         df_categories.drop(columns=[ReportKey.category_name], inplace=True)
         return df_categories