evalscope 1.0.0__py3-none-any.whl → 1.2.0__py3-none-any.whl

evalscope/report/__init__.py

@@ -4,9 +4,16 @@ from typing import TYPE_CHECKING
 from evalscope.utils.import_utils import _LazyModule
 
 if TYPE_CHECKING:
-    from .combinator import gen_table, get_data_frame, get_report_list
+    from .combinator import (
+        gen_table,
+        get_data_frame,
+        get_report_list,
+        percentage_weighted_average_from_subsets,
+        unweighted_average_from_subsets,
+        weighted_average_from_subsets,
+    )
     from .generator import ReportGenerator
-    from .report import Category, Report, ReportKey, Subset
+    from .report import Category, Metric, Report, ReportKey, Subset
 
 else:
     _import_structure = {
@@ -14,7 +21,9 @@ else:
             'gen_table',
             'get_data_frame',
             'get_report_list',
-            'gen_report_table',
+            'weighted_average_from_subsets',
+            'unweighted_average_from_subsets',
+            'percentage_weighted_average_from_subsets',
         ],
         'generator': [
             'ReportGenerator',
@@ -24,6 +33,7 @@ else:
             'Report',
             'ReportKey',
             'Subset',
+            'Metric',
         ],
     }
 

evalscope/report/combinator.py

@@ -4,9 +4,9 @@ import glob
 import os
 import pandas as pd
 from tabulate import tabulate
-from typing import List, Tuple
+from typing import Dict, List, Tuple, Union
 
-from evalscope.report.report import Report
+from evalscope.report.report import Report, Subset
 from evalscope.utils.logger import get_logger
 
 logger = get_logger()
@@ -88,26 +88,97 @@ def gen_table(
     return tabulate(table, headers=table.columns, tablefmt='grid', showindex=False)
 
 
-class ReportsRecorder:
-    COMMON_DATASET_PATH = []
-    CUSTOM_DATASET_PATH = []
+def weighted_average_from_subsets(
+    subset_names: List[str], subset_dict: Dict[str, Subset], new_name: str = ''
+) -> Subset:
+    """Calculate weighted average for given subsets.
 
-    def __init__(self, oss_url: str = '', endpoint: str = ''):
-        pass
+    Args:
+        subset_names (List[str]): List of subset names to include in the average.
+        subset_dict (Dict[str, Subset]): Dictionary mapping subset names to Subset objects.
+        new_name (str): Name for the resulting Subset object.
+
+    Returns:
+        Subset: A new Subset object with weighted average score
+    """
+    total_score = 0
+    total_count = 0
+    for name in subset_names:
+        if name in subset_dict:
+            subset = subset_dict[name]
+            total_score += subset.score * subset.num
+            total_count += subset.num
+
+    weighted_avg = total_score / total_count if total_count > 0 else 0
+    return Subset(name=new_name, score=weighted_avg, num=total_count)
+
+
+def unweighted_average_from_subsets(
+    subset_names: List[str], subset_dict: Dict[str, Subset], new_name: str = ''
+) -> Subset:
+    """Calculate unweighted average for given subsets.
+
+    Args:
+        subset_names (List[str]): List of subset names to include in the average.
+        subset_dict (Dict[str, Subset]): Dictionary mapping subset names to Subset objects.
+        new_name (str): Name for the resulting Subset object.
+
+    Returns:
+        Subset: A new Subset object with unweighted average score
+    """
+    scores = []
+    total_count = 0
+    for name in subset_names:
+        if name in subset_dict:
+            subset = subset_dict[name]
+            scores.append(subset.score)
+            total_count += subset.num
+
+    unweighted_avg = sum(scores) / len(scores) if scores else 0
+    return Subset(name=new_name, score=unweighted_avg, num=total_count)
+
+
+def percentage_weighted_average_from_subsets(
+    subset_names: List[str], subset_dict: Dict[str, Subset], weights: List[float], new_name: str = ''
+) -> Subset:
+    """Calculate percentage weighted average for given subsets.
+
+    Args:
+        subset_names (List[str]): List of subset names to include in the average.
+        subset_dict (Dict[str, Subset]): Dictionary mapping subset names to Subset objects.
+        weights (List[float]): The weight for each corresponding accuracy entry.
+            Can sum to any positive value – they will be normalised internally.
+        new_name (str): Name for the resulting Subset object.
+
+    Returns:
+        Subset: A new Subset object with percentage weighted average score.
+    """
+    assert len(subset_names) == len(weights), \
+        'The number of subset names must match the number of weights.'
+
+    valid_subsets = []
+    valid_weights = []
+    total_count = 0
+
+    for name, weight in zip(subset_names, weights):
+        if name in subset_dict:
+            subset = subset_dict[name]
+            valid_subsets.append(subset)
+            valid_weights.append(weight)
+            total_count += subset.num
+
+    if not valid_subsets:
+        return Subset(name=new_name, score=0, num=0)
 
+    weight_sum = sum(valid_weights)
+    assert weight_sum > 0, \
+        f"Sum of weights for percentage_weighted_average_from_subsets for '{new_name}' is not positive."
 
-if __name__ == '__main__':
-    report_dir_1 = './outputs/20250117_151926'
-    # report_dir_2 = './outputs/20250107_204445/reports'
+    # Normalise weights so that they sum to 1.0
+    weights_norm = [w / weight_sum for w in valid_weights]
 
-    report_table = gen_table(reports_path_list=[report_dir_1])
-    print(report_table)
+    total_score = 0
+    for subset, weight in zip(valid_subsets, weights_norm):
+        total_score += subset.score * weight
 
-    # ALL VALUES ONLY FOR EXAMPLE
-    # +--------------------------+-------------------+-------------+
-    # | Model                    | CompetitionMath   | GSM8K       |
-    # +==========================+===================+=============+
-    # | ZhipuAI_chatglm2-6b-base | 25.0 (acc)        | 30.50 (acc) |
-    # +--------------------------+-------------------+-------------+
-    # | ZhipuAI_chatglm2-6b      | 30.5 (acc)        | 40.50 (acc) |
-    # +--------------------------+-------------------+-------------+
+    return Subset(name=new_name, score=total_score, num=total_count)

evalscope/report/generator.py

@@ -8,105 +8,26 @@ from evalscope.report.report import *
 if TYPE_CHECKING:
     from evalscope.api.benchmark import DataAdapter
     from evalscope.api.metric import AggScore
-    from evalscope.benchmarks import DataAdapter as OldDataAdapter
 
 
 class ReportGenerator:
 
     @staticmethod
-    def gen_report(subset_score_map: dict, model_name: str, data_adapter: 'OldDataAdapter', **kwargs) -> Report:
-        """
-        Generate a report for a specific dataset based on provided subset scores.
-
-        Args:
-            subset_score_map (dict): A mapping from subset names to a list of score dictionaries.
-                    {
-                        'subset_name': [
-                            {'metric_name': 'AverageAccuracy', 'score': 0.3389, 'num': 100},
-                            {'metric_name': 'WeightedAverageAccuracy', 'score': 0.3389, 'num': 100}
-                        ],
-                        ...
-                    }
-            report_name (str): The name of the report to generate.
-            data_adapter (DataAdapter): An adapter object for data handling.
-
-        Returns:
-            Report: A structured report object containing metrics, categories, and subsets.
-
-            >>> report = gen_report(subset_score_map, "My Report", data_adapter, dataset_name="Dataset", model_name="Model")
-        """  # noqa: E501
-
-        dataset_name = data_adapter.name
-        category_map = data_adapter.category_map
-        report_name = f'{model_name}@{dataset_name}'
-
-        def flatten_subset() -> DataFrame:
-            """
-            Flatten subset score map to a DataFrame.
-
-            Example:
-                        name  score  num   categories      metric_name
-            0       ARC-Easy    0.5    2    [default]  AverageAccuracy
-            1  ARC-Challenge    0.5    2    [default]  AverageAccuracy
-            """
-            subsets = []
-            for subset_name, scores in subset_score_map.items():
-                for score_item in scores:
-                    categories = category_map.get(subset_name, ['default'])
-                    if isinstance(categories, str):
-                        categories = [categories]
-                    subsets.append(
-                        dict(
-                            name=subset_name,
-                            score=score_item['score'],
-                            num=score_item['num'],
-                            metric_name=score_item['metric_name'],
-                            categories=tuple(categories)
-                        )
-                    )
-            df = pd.DataFrame(subsets)
-            return df
-
-        df = flatten_subset()
-
+    def gen_collection_report(df: DataFrame, all_dataset_name: str, model_name: str) -> Report:
         metrics_list = []
-        for metric_name, group_metric in df.groupby('metric_name', sort=False):
+        for metric_name, group_metric in df.groupby('metric', sort=False):
             categories = []
             for category_name, group_category in group_metric.groupby('categories'):
                 subsets = []
-                for _, row in group_category.iterrows():
-                    subsets.append(Subset(name=row['name'], score=row['score'], num=row['num']))
-
+                for (dataset_name, subset_name), group_subset in group_category.groupby(['dataset_name',
+                                                                                         'subset_name']):
+                    avg_score = group_subset['score'].mean()
+                    num = group_subset['score'].count()
+                    subsets.append(Subset(name=f'{dataset_name}/{subset_name}', score=float(avg_score), num=int(num)))
                 categories.append(Category(name=category_name, subsets=subsets))
-
             metrics_list.append(Metric(name=metric_name, categories=categories))
-
-        report = Report(
-            name=report_name,
-            metrics=metrics_list,
-            dataset_name=dataset_name,
-            model_name=model_name,
-            dataset_description=data_adapter.description,
-            dataset_pretty_name=data_adapter.pretty_name
-        )
-        return report
-
-    @staticmethod
-    def gen_collection_report(df: DataFrame, all_dataset_name: str, model_name: str) -> Report:
-        categories = []
-        for category_name, group_category in df.groupby('categories'):
-            subsets = []
-            for (dataset_name, subset_name), group_subset in group_category.groupby(['dataset_name', 'subset_name']):
-                avg_score = group_subset['score'].mean()
-                num = group_subset['score'].count()
-                subsets.append(Subset(name=f'{dataset_name}/{subset_name}', score=float(avg_score), num=int(num)))
-
-            categories.append(Category(name=category_name, subsets=subsets))
         return Report(
-            name=DataCollection.NAME,
-            metrics=[Metric(name='Average', categories=categories)],
-            dataset_name=all_dataset_name,
-            model_name=model_name
+            name=DataCollection.NAME, metrics=metrics_list, dataset_name=all_dataset_name, model_name=model_name
         )
 
     @staticmethod

evalscope/report/report.py

@@ -22,7 +22,7 @@ ANALYSIS_PROMPT = """根据给出的json格式的模型评测结果，输出分
 """
 
 
-def normalize_score(score: Union[float, dict], keep_num: int = 4) -> Union[float, dict]:
+def normalize_score(score: Union[float, dict, int], keep_num: int = 4) -> Union[float, dict]:
     """
     Normalize score.
 
@@ -37,9 +37,10 @@ def normalize_score(score: Union[float, dict], keep_num: int = 4) -> Union[float
         score = round(score, keep_num)
     elif isinstance(score, dict):
         score = {k: round(v, keep_num) for k, v in score.items()}
+    elif isinstance(score, int):
+        score = float(score)
     else:
         logger.warning(f'Unknown score type: {type(score)}')
-
     return score
 
 
@@ -103,6 +104,7 @@ class ReportKey:
     subset_name = 'Subset'
     num = 'Num'
     score = 'Score'
+    overall_score = 'OVERALL'
 
 
 @dataclass
@@ -181,12 +183,14 @@ class Report:
                     table[ReportKey.num].append(subset.num)
                     table[ReportKey.score].append(subset.score)
             # add overall metric when there are multiple subsets
-            if metric_count > 1 and add_overall_metric:
+            if metric_count > 1 and add_overall_metric and (
+                ReportKey.overall_score not in table[ReportKey.subset_name]
+            ):
                 table[ReportKey.model_name].append(self.model_name)
                 table[ReportKey.dataset_name].append(self.dataset_name)
                 table[ReportKey.metric_name].append(metric.name)
                 table[ReportKey.category_name].append(('-', ))
-                table[ReportKey.subset_name].append('OVERALL')
+                table[ReportKey.subset_name].append(ReportKey.overall_score)
                 table[ReportKey.num].append(metric.num)
                 table[ReportKey.score].append(metric.score)
             # NOTE: only flatten metrics if needed, use the first metric by default

evalscope/run.py

@@ -38,6 +38,7 @@ def run_single_task(task_cfg: TaskConfig, run_time: str) -> dict:
     if task_cfg.eval_backend != EvalBackend.NATIVE:
         result = run_non_native_backend(task_cfg, outputs)
     else:
+        logger.info('Running with native backend')
         result = evaluate_model(task_cfg, outputs)
 
         logger.info(f'Finished evaluation for {task_cfg.model_id} on {task_cfg.datasets}')
@@ -94,12 +95,15 @@ def run_non_native_backend(task_cfg: TaskConfig, outputs: OutputsStructure) -> d
 def get_backend_manager_class(eval_backend: EvalBackend):
     """Get the backend manager class based on the evaluation backend."""
     if eval_backend == EvalBackend.OPEN_COMPASS:
+        logger.info('Using OpenCompassBackendManager')
         from evalscope.backend.opencompass import OpenCompassBackendManager
         return OpenCompassBackendManager
     elif eval_backend == EvalBackend.VLM_EVAL_KIT:
+        logger.info('Using VLMEvalKitBackendManager')
         from evalscope.backend.vlm_eval_kit import VLMEvalKitBackendManager
         return VLMEvalKitBackendManager
     elif eval_backend == EvalBackend.RAG_EVAL:
+        logger.info('Using RAGEvalBackendManager')
         from evalscope.backend.rag_eval import RAGEvalBackendManager
         return RAGEvalBackendManager
     elif eval_backend == EvalBackend.THIRD_PARTY:
@@ -131,8 +135,9 @@ def evaluate_model(task_config: TaskConfig, outputs: OutputsStructure) -> dict:
         )
         evaluators.append(evaluator)
 
-        # Update task_config.dataset_args with benchmark metadata
-        task_config.dataset_args[dataset_name] = benchmark.to_dict()
+        # Update task_config.dataset_args with benchmark metadata, except for DataCollection
+        if dataset_name != DataCollection.NAME:
+            task_config.dataset_args[dataset_name] = benchmark.to_dict()
 
     # dump task_cfg to outputs.configs_dir after creating evaluators
     task_config.dump_yaml(outputs.configs_dir)
@@ -149,17 +154,20 @@ def evaluate_model(task_config: TaskConfig, outputs: OutputsStructure) -> dict:
         logger.info(f'Overall report table: \n{report_table} \n')
     except Exception:
         logger.error('Failed to generate report table.')
-
     # Clean up
     if model is not None:
         import gc
-        import torch
 
         del model
         del evaluators
-        torch.cuda.empty_cache()
         gc.collect()
 
+        from evalscope.utils.import_utils import check_import
+        if check_import('torch', raise_warning=False):
+            import torch
+            if torch.cuda.is_available():
+                torch.cuda.empty_cache()
+
     return eval_results
 
 

evalscope/third_party/toolbench_static/llm/swift_infer.py

@@ -1,9 +1,5 @@
-import os
 from dataclasses import dataclass
-from swift.llm import InferEngine, InferRequest, PtEngine, RequestConfig, get_template
 
-# 设置GPU环境变量
-os.environ['CUDA_VISIBLE_DEVICES'] = '0'
 
 @dataclass
 class SwiftInferArgs:

evalscope/utils/argument_utils.py

@@ -61,4 +61,4 @@ def parse_int_or_float(num):
 def get_supported_params(func):
     """Get the supported parameters of a function."""
     sig = signature(func)
-    return list(sig.parameters.keys())
+    return set(sig.parameters.keys())

evalscope/utils/chat_service.py

@@ -1,6 +1,5 @@
 import os
 import time
-import torch
 from contextlib import contextmanager
 from functools import partial
 from pydantic import BaseModel, Field
@@ -95,6 +94,7 @@ class TextCompletionResponse(BaseModel):
 class ChatService:
 
     def __init__(self, model_path, attn_implementation):
+        import torch
         from modelscope import AutoModelForCausalLM, AutoTokenizer
         from transformers import TextIteratorStreamer