evalscope 0.10.0__py3-none-any.whl → 1.2.0__py3-none-any.whl

evalscope/report/__init__.py

@@ -1,5 +1,48 @@
 # Copyright (c) Alibaba, Inc. and its affiliates.
+from typing import TYPE_CHECKING
 
-from evalscope.report.combinator import gen_table, get_data_frame, get_report_list
-from evalscope.report.generator import ReportGenerator
-from evalscope.report.utils import Category, Report, ReportKey, Subset
+from evalscope.utils.import_utils import _LazyModule
+
+if TYPE_CHECKING:
+    from .combinator import (
+        gen_table,
+        get_data_frame,
+        get_report_list,
+        percentage_weighted_average_from_subsets,
+        unweighted_average_from_subsets,
+        weighted_average_from_subsets,
+    )
+    from .generator import ReportGenerator
+    from .report import Category, Metric, Report, ReportKey, Subset
+
+else:
+    _import_structure = {
+        'combinator': [
+            'gen_table',
+            'get_data_frame',
+            'get_report_list',
+            'weighted_average_from_subsets',
+            'unweighted_average_from_subsets',
+            'percentage_weighted_average_from_subsets',
+        ],
+        'generator': [
+            'ReportGenerator',
+        ],
+        'report': [
+            'Category',
+            'Report',
+            'ReportKey',
+            'Subset',
+            'Metric',
+        ],
+    }
+
+    import sys
+
+    sys.modules[__name__] = _LazyModule(
+        __name__,
+        globals()['__file__'],
+        _import_structure,
+        module_spec=__spec__,
+        extra_objects={},
+    )

evalscope/report/combinator.py

@@ -4,9 +4,9 @@ import glob
 import os
 import pandas as pd
 from tabulate import tabulate
-from typing import List, Tuple
+from typing import Dict, List, Tuple, Union
 
-from evalscope.report.utils import Report
+from evalscope.report.report import Report, Subset
 from evalscope.utils.logger import get_logger
 
 logger = get_logger()
@@ -32,42 +32,153 @@ def get_report_list(reports_path_list: List[str]) -> List[Report]:
     return report_list
 
 
-def get_data_frame(report_list: List[Report],
-                   flatten_metrics: bool = True,
-                   flatten_categories: bool = True) -> pd.DataFrame:
+def get_data_frame(
+    report_list: List[Report],
+    flatten_metrics: bool = True,
+    flatten_categories: bool = True,
+    add_overall_metric: bool = False
+) -> pd.DataFrame:
     tables = []
     for report in report_list:
-        df = report.to_dataframe(flatten_metrics=flatten_metrics, flatten_categories=flatten_categories)
+        df = report.to_dataframe(
+            flatten_metrics=flatten_metrics,
+            flatten_categories=flatten_categories,
+            add_overall_metric=add_overall_metric
+        )
         tables.append(df)
     return pd.concat(tables, ignore_index=True)
 
 
-def gen_table(reports_path_list: list) -> str:
-    report_list = get_report_list(reports_path_list)
-    table = get_data_frame(report_list)
+def gen_table(
+    reports_path_list: list[str] = None,
+    report_list: list[Report] = None,
+    flatten_metrics: bool = True,
+    flatten_categories: bool = True,
+    add_overall_metric: bool = False
+) -> str:
+    """
+    Generates a formatted table from a list of report paths or Report objects.
+
+    Args:
+        reports_path_list (list[str], optional): List of file paths to report files.
+            Either this or `report_list` must be provided.
+        report_list (list[Report], optional): List of Report objects.
+            Either this or `reports_path_list` must be provided.
+        flatten_metrics (bool, optional): Whether to flatten the metrics in the output table. Defaults to True.
+        flatten_categories (bool, optional): Whether to flatten the categories in the output table. Defaults to True.
+        add_overall_metric (bool, optional): Whether to add an overall metric column to the table. Defaults to False.
+
+    Returns:
+        str: A string representation of the table in grid format.
+
+    Raises:
+        AssertionError: If neither `reports_path_list` nor `report_list` is provided.
+    """
+    assert (reports_path_list is not None) or (report_list is not None), \
+        'Either reports_path_list or report_list must be provided.'
+    if report_list is None:
+        report_list = get_report_list(reports_path_list)
+    # Generate a DataFrame from the report list
+    table = get_data_frame(
+        report_list,
+        flatten_metrics=flatten_metrics,
+        flatten_categories=flatten_categories,
+        add_overall_metric=add_overall_metric
+    )
     return tabulate(table, headers=table.columns, tablefmt='grid', showindex=False)
 
 
-class ReportsRecorder:
-    COMMON_DATASET_PATH = []
-    CUSTOM_DATASET_PATH = []
-
-    def __init__(self, oss_url: str = '', endpoint: str = ''):
-        pass
-
-
-if __name__ == '__main__':
-    report_dir_1 = '/mnt/data/data/user/maoyunlin.myl/eval-scope/outputs/20250117_151926'
-    # report_dir_2 = '/mnt/data/data/user/maoyunlin.myl/eval-scope/outputs/20250107_204445/reports'
-
-    report_table = gen_table([report_dir_1])
-    print(report_table)
-
-    # ALL VALUES ONLY FOR EXAMPLE
-    # +--------------------------+-------------------+-------------+
-    # | Model                    | CompetitionMath   | GSM8K       |
-    # +==========================+===================+=============+
-    # | ZhipuAI_chatglm2-6b-base | 25.0 (acc)        | 30.50 (acc) |
-    # +--------------------------+-------------------+-------------+
-    # | ZhipuAI_chatglm2-6b      | 30.5 (acc)        | 40.50 (acc) |
-    # +--------------------------+-------------------+-------------+
+def weighted_average_from_subsets(
+    subset_names: List[str], subset_dict: Dict[str, Subset], new_name: str = ''
+) -> Subset:
+    """Calculate weighted average for given subsets.
+
+    Args:
+        subset_names (List[str]): List of subset names to include in the average.
+        subset_dict (Dict[str, Subset]): Dictionary mapping subset names to Subset objects.
+        new_name (str): Name for the resulting Subset object.
+
+    Returns:
+        Subset: A new Subset object with weighted average score
+    """
+    total_score = 0
+    total_count = 0
+    for name in subset_names:
+        if name in subset_dict:
+            subset = subset_dict[name]
+            total_score += subset.score * subset.num
+            total_count += subset.num
+
+    weighted_avg = total_score / total_count if total_count > 0 else 0
+    return Subset(name=new_name, score=weighted_avg, num=total_count)
+
+
+def unweighted_average_from_subsets(
+    subset_names: List[str], subset_dict: Dict[str, Subset], new_name: str = ''
+) -> Subset:
+    """Calculate unweighted average for given subsets.
+
+    Args:
+        subset_names (List[str]): List of subset names to include in the average.
+        subset_dict (Dict[str, Subset]): Dictionary mapping subset names to Subset objects.
+        new_name (str): Name for the resulting Subset object.
+
+    Returns:
+        Subset: A new Subset object with unweighted average score
+    """
+    scores = []
+    total_count = 0
+    for name in subset_names:
+        if name in subset_dict:
+            subset = subset_dict[name]
+            scores.append(subset.score)
+            total_count += subset.num
+
+    unweighted_avg = sum(scores) / len(scores) if scores else 0
+    return Subset(name=new_name, score=unweighted_avg, num=total_count)
+
+
+def percentage_weighted_average_from_subsets(
+    subset_names: List[str], subset_dict: Dict[str, Subset], weights: List[float], new_name: str = ''
+) -> Subset:
+    """Calculate percentage weighted average for given subsets.
+
+    Args:
+        subset_names (List[str]): List of subset names to include in the average.
+        subset_dict (Dict[str, Subset]): Dictionary mapping subset names to Subset objects.
+        weights (List[float]): The weight for each corresponding accuracy entry.
+            Can sum to any positive value – they will be normalised internally.
+        new_name (str): Name for the resulting Subset object.
+
+    Returns:
+        Subset: A new Subset object with percentage weighted average score.
+    """
+    assert len(subset_names) == len(weights), \
+        'The number of subset names must match the number of weights.'
+
+    valid_subsets = []
+    valid_weights = []
+    total_count = 0
+
+    for name, weight in zip(subset_names, weights):
+        if name in subset_dict:
+            subset = subset_dict[name]
+            valid_subsets.append(subset)
+            valid_weights.append(weight)
+            total_count += subset.num
+
+    if not valid_subsets:
+        return Subset(name=new_name, score=0, num=0)
+
+    weight_sum = sum(valid_weights)
+    assert weight_sum > 0, \
+        f"Sum of weights for percentage_weighted_average_from_subsets for '{new_name}' is not positive."
+
+    # Normalise weights so that they sum to 1.0
+    weights_norm = [w / weight_sum for w in valid_weights]
+
+    total_score = 0
+    for subset, weight in zip(valid_subsets, weights_norm):
+        total_score += subset.score * weight
+
+    return Subset(name=new_name, score=total_score, num=total_count)

evalscope/report/generator.py

@@ -1,24 +1,67 @@
 import pandas as pd
 from pandas import DataFrame
+from typing import TYPE_CHECKING
 
 from evalscope.constants import DataCollection
-from evalscope.report.utils import *
+from evalscope.report.report import *
+
+if TYPE_CHECKING:
+    from evalscope.api.benchmark import DataAdapter
+    from evalscope.api.metric import AggScore
 
 
 class ReportGenerator:
 
     @staticmethod
-    def gen_report(subset_score_map: dict, report_name: str, **kwargs) -> Report:
+    def gen_collection_report(df: DataFrame, all_dataset_name: str, model_name: str) -> Report:
+        metrics_list = []
+        for metric_name, group_metric in df.groupby('metric', sort=False):
+            categories = []
+            for category_name, group_category in group_metric.groupby('categories'):
+                subsets = []
+                for (dataset_name, subset_name), group_subset in group_category.groupby(['dataset_name',
+                                                                                         'subset_name']):
+                    avg_score = group_subset['score'].mean()
+                    num = group_subset['score'].count()
+                    subsets.append(Subset(name=f'{dataset_name}/{subset_name}', score=float(avg_score), num=int(num)))
+                categories.append(Category(name=category_name, subsets=subsets))
+            metrics_list.append(Metric(name=metric_name, categories=categories))
+        return Report(
+            name=DataCollection.NAME, metrics=metrics_list, dataset_name=all_dataset_name, model_name=model_name
+        )
+
+    @staticmethod
+    def generate_report(
+        score_dict: Dict[str, List['AggScore']],
+        model_name: str,
+        data_adapter: 'DataAdapter',
+        add_aggregation_name: bool = True
+    ) -> Report:
         """
-        Generate report for specific dataset.
-        subset_score_map: e.g. {subset_name: [{'metric_name': 'AverageAccuracy', 'score': 0.3389, 'num': 100}, {'metric_name': 'WeightedAverageAccuracy', 'score': 0.3389, 'num': 100}]}
-        category_map: e.g. {'subset_name': ['category_name1', 'category_name2'], ...}
-        metric_list: e.g. [{'object': AverageAccuracy, 'name': 'AverageAccuracy'}, {'object': 'WeightedAverageAccuracy', 'name': 'WeightedAverageAccuracy'}]
+        Generate a report for a specific dataset based on provided subset scores.
+
+        Args:
+            subset_score_map (dict): A mapping from subset names to a list of score dictionaries.
+            ```
+            {
+                'subset_name': [
+                    AggScore={'metric_name': 'AverageAccuracy', 'score': 0.3389, 'num': 100},
+                    AggScore={'metric_name': 'WeightedAverageAccuracy', 'score': 0.3389, 'num': 100}
+                ],
+                ...
+            }
+            ```
+            data_adapter (DataAdapter): An adapter object for data handling.
+
+        Returns:
+            Report: A structured report object containing metrics, categories, and subsets.
+
+            >>> report = gen_report(subset_score_map, "My Report", data_adapter, dataset_name="Dataset", model_name="Model")
         """  # noqa: E501
 
-        dataset_name = kwargs.get('dataset_name', None)
-        model_name = kwargs.get('model_name', None)
-        category_map = kwargs.get('category_map', {})
+        dataset_name = data_adapter.name
+        category_map = data_adapter.category_map
+        report_name = f'{model_name}@{dataset_name}'
 
         def flatten_subset() -> DataFrame:
             """
@@ -30,25 +73,32 @@ class ReportGenerator:
             1  ARC-Challenge    0.5    2    [default]  AverageAccuracy
             """
             subsets = []
-            for subset_name, scores in subset_score_map.items():
-                for score_item in scores:
+            for subset_name, agg_scores in score_dict.items():
+                for agg_score_item in agg_scores:
                     categories = category_map.get(subset_name, ['default'])
+                    if add_aggregation_name and agg_score_item.aggregation_name:
+                        metric_name = f'{agg_score_item.aggregation_name}_{agg_score_item.metric_name}'
+                    else:
+                        metric_name = agg_score_item.metric_name
+
                     if isinstance(categories, str):
                         categories = [categories]
                     subsets.append(
                         dict(
                             name=subset_name,
-                            score=score_item['score'],
-                            num=score_item['num'],
-                            metric_name=score_item['metric_name'],
-                            categories=tuple(categories)))
+                            score=agg_score_item.score,
+                            num=agg_score_item.num,
+                            metric_name=metric_name,
+                            categories=tuple(categories)
+                        )
+                    )
             df = pd.DataFrame(subsets)
             return df
 
         df = flatten_subset()
 
         metrics_list = []
-        for metric_name, group_metric in df.groupby('metric_name'):
+        for metric_name, group_metric in df.groupby('metric_name', sort=False):
             categories = []
             for category_name, group_category in group_metric.groupby('categories'):
                 subsets = []
@@ -59,22 +109,12 @@ class ReportGenerator:
 
             metrics_list.append(Metric(name=metric_name, categories=categories))
 
-        report = Report(name=report_name, metrics=metrics_list, dataset_name=dataset_name, model_name=model_name)
+        report = Report(
+            name=report_name,
+            metrics=metrics_list,
+            dataset_name=dataset_name,
+            model_name=model_name,
+            dataset_description=data_adapter.description,
+            dataset_pretty_name=data_adapter.pretty_name
+        )
         return report
-
-    @staticmethod
-    def gen_collection_report(df: DataFrame, all_dataset_name: str, model_name: str) -> Report:
-        categories = []
-        for category_name, group_category in df.groupby('categories'):
-            subsets = []
-            for (dataset_name, subset_name), group_subset in group_category.groupby(['dataset_name', 'subset_name']):
-                avg_score = group_subset['score'].mean()
-                num = group_subset['score'].count()
-                subsets.append(Subset(name=f'{dataset_name}/{subset_name}', score=float(avg_score), num=int(num)))
-
-            categories.append(Category(name=category_name, subsets=subsets))
-        return Report(
-            name=DataCollection.NAME,
-            metrics=[Metric(name='Average', categories=categories)],
-            dataset_name=all_dataset_name,
-            model_name=model_name)

evalscope/report/report.py

@@ -0,0 +1,238 @@
+import json
+import os
+import pandas as pd
+from collections import defaultdict
+from dataclasses import asdict, dataclass, field
+from typing import Any, Dict, List, Union
+
+from evalscope.metrics import macro_mean, micro_mean
+from evalscope.utils import get_logger
+
+logger = get_logger()
+
+ANALYSIS_PROMPT = """根据给出的json格式的模型评测结果，输出分析报告，要求如下：
+1. 报告分为 总体表现、关键指标分析、改进建议、结论 四部分
+2. 若模型有多种指标，将其分为低分、中分、高分三个部分，并列出markdown表格
+3. 只列出报告本身，不要有其他多余内容
+4. 输出报告语言为{language}
+
+```json
+{report_str}
+```
+"""
+
+
+def normalize_score(score: Union[float, dict, int], keep_num: int = 4) -> Union[float, dict]:
+    """
+    Normalize score.
+
+    Args:
+        score: input score, could be float or dict. e.g. 0.12345678 or {'acc': 0.12345678, 'f1': 0.12345678}
+        keep_num: number of digits to keep.
+
+    Returns:
+        Union[float, dict]: normalized score. e.g. 0.1234 or {'acc': 0.1234, 'f1': 0.1234}
+    """
+    if isinstance(score, float):
+        score = round(score, keep_num)
+    elif isinstance(score, dict):
+        score = {k: round(v, keep_num) for k, v in score.items()}
+    elif isinstance(score, int):
+        score = float(score)
+    else:
+        logger.warning(f'Unknown score type: {type(score)}')
+    return score
+
+
+@dataclass
+class Subset:
+    name: str = 'default_subset'
+    score: float = 0.0
+    num: int = 0
+
+    def __post_init__(self):
+        self.score = normalize_score(self.score)
+
+
+@dataclass
+class Category:
+    name: tuple[str] = field(default_factory=tuple)
+    num: int = 0
+    score: float = 0.0
+    macro_score: float = 0.0
+    subsets: List[Subset] = field(default_factory=list)
+
+    def __post_init__(self):
+        if isinstance(self.name, str):
+            # ensure name is tuple format
+            self.name = (self.name, )
+        self.num = sum(subset.num for subset in self.subsets)
+        self.score = normalize_score(micro_mean(self.subsets))
+        self.macro_score = normalize_score(macro_mean(self.subsets))
+
+    @classmethod
+    def from_dict(cls, data: dict):
+        subsets = [Subset(**subset) for subset in data.get('subsets', [])]
+        return cls(name=data['name'], subsets=subsets)
+
+
+@dataclass
+class Metric:
+    name: str = 'default_metric'
+    num: int = 0
+    score: float = 0.0
+    macro_score: float = 0.0
+    categories: List[Category] = field(default_factory=list)
+
+    def __post_init__(self):
+        self.num = sum(category.num for category in self.categories)
+        self.score = normalize_score(micro_mean(self.categories))
+        self.macro_score = normalize_score(macro_mean(self.categories))
+
+    @classmethod
+    def from_dict(cls, data: dict):
+        categories = [Category.from_dict(category) for category in data.get('categories', [])]
+        return cls(name=data['name'], categories=categories)
+
+
+class ReportKey:
+    model_name = 'Model'
+    dataset_name = 'Dataset'
+    metric_name = 'Metric'
+    category_name = 'Category'
+    category_prefix = 'Cat.'
+    subset_name = 'Subset'
+    num = 'Num'
+    score = 'Score'
+    overall_score = 'OVERALL'
+
+
+@dataclass
+class Report:
+    name: str = 'default_report'
+    dataset_name: str = 'default_dataset'
+    dataset_pretty_name: str = ''
+    dataset_description: str = ''
+    model_name: str = 'default_model'
+    score: float = 0.0
+    metrics: List[Metric] = field(default_factory=list)
+    analysis: str = 'N/A'
+
+    def __post_init__(self):
+        self.score = self.metrics[0].score  # NOTE: only use the first metric by default
+
+    def to_dict(self) -> Dict[str, Any]:
+        return asdict(self)
+
+    def to_json_str(self) -> str:
+        return json.dumps(self.to_dict(), indent=4, ensure_ascii=False)
+
+    def to_json(self, json_file: str):
+        # ensure the directory exists
+        os.makedirs(os.path.dirname(json_file), exist_ok=True)
+        # write the report to a json file
+        with open(json_file, 'w', encoding='utf-8') as f:
+            json.dump(self.to_dict(), f, indent=4, ensure_ascii=False)
+
+    @classmethod
+    def from_dict(cls, data: dict):
+        metrics = [Metric.from_dict(metric) for metric in data.get('metrics', [])]
+        return cls(
+            name=data['name'],
+            dataset_name=data['dataset_name'],
+            dataset_pretty_name=data.get('dataset_pretty_name'),
+            dataset_description=data.get('dataset_description'),
+            score=data['score'],
+            model_name=data['model_name'],
+            metrics=metrics,
+            analysis=data.get('analysis', 'N/A'),
+        )
+
+    @classmethod
+    def from_json(cls, json_file: str):
+        with open(json_file, 'r', encoding='utf-8') as f:
+            data = json.load(f)
+        return cls.from_dict(data)
+
+    def to_dataframe(
+        self,
+        flatten_metrics: bool = True,
+        flatten_categories: bool = True,
+        add_overall_metric: bool = False
+    ) -> pd.DataFrame:
+        """
+        Convert the report to a pandas DataFrame.
+        Args:
+            flatten_metrics (bool): Whether to flatten the metrics to a single row.
+            flatten_categories (bool): Whether to flatten the categories to multiple rows.
+            add_overall_metric (bool): Whether to add an overall metric row.
+        Returns:
+            pd.DataFrame: The report as a pandas DataFrame.
+        """
+        table = defaultdict(list)
+        for metric in self.metrics:
+            metric_count = 0
+            for category in metric.categories:
+                for subset in category.subsets:
+                    metric_count += 1
+                    table[ReportKey.model_name].append(self.model_name)
+                    table[ReportKey.dataset_name].append(self.dataset_name)
+                    table[ReportKey.metric_name].append(metric.name)
+                    table[ReportKey.category_name].append(category.name)
+                    table[ReportKey.subset_name].append(subset.name)
+                    table[ReportKey.num].append(subset.num)
+                    table[ReportKey.score].append(subset.score)
+            # add overall metric when there are multiple subsets
+            if metric_count > 1 and add_overall_metric and (
+                ReportKey.overall_score not in table[ReportKey.subset_name]
+            ):
+                table[ReportKey.model_name].append(self.model_name)
+                table[ReportKey.dataset_name].append(self.dataset_name)
+                table[ReportKey.metric_name].append(metric.name)
+                table[ReportKey.category_name].append(('-', ))
+                table[ReportKey.subset_name].append(ReportKey.overall_score)
+                table[ReportKey.num].append(metric.num)
+                table[ReportKey.score].append(metric.score)
+            # NOTE: only flatten metrics if needed, use the first metric by default
+            if not flatten_metrics:
+                break
+        df = pd.DataFrame.from_dict(table, orient='columns')
+        if flatten_categories:
+            df = self._flatten_categories(df)
+        return df
+
+    def _flatten_categories(self, df: pd.DataFrame):
+        # expand categories to multiple rows
+        df_categories = df.copy()
+        # multi-level aggregation for categories
+        max_depth = df_categories[ReportKey.category_name].apply(len).max()
+        for level in range(max_depth):
+            df_categories[f'{ReportKey.category_prefix}{level}'] = df_categories[
+                ReportKey.category_name].apply(lambda x: x[level] if len(x) > level else None)
+
+        df_categories.drop(columns=[ReportKey.category_name], inplace=True)
+        return df_categories
+
+    def generate_analysis(self, judge_llm_config: dict) -> str:
+        import locale
+
+        from evalscope.metrics import LLMJudge
+
+        try:
+            # get the default locale
+            lang, _ = locale.getlocale()
+
+            if lang is None:
+                language = '中文'
+            else:
+                language = 'en' if lang.startswith('en') else '中文'
+
+            prompt = ANALYSIS_PROMPT.format(language=language, report_str=self.to_json_str())
+            judge_llm = LLMJudge(**judge_llm_config)
+            response = judge_llm(prompt)
+        except Exception as e:
+            logger.error(f'Error generating analysis: {e}')
+            response = 'N/A'
+
+        self.analysis = response
+        return response