evalscope 0.15.1__py3-none-any.whl → 0.16.1__py3-none-any.whl

evalscope/perf/utils/rich_display.py

@@ -0,0 +1,186 @@
+# the following code is largely adapted from https://github.com/lework/llm-benchmark
+
+import numpy as np
+from rich.console import Console
+from rich.panel import Panel
+from rich.style import Style
+from rich.table import Table
+from rich.text import Text
+
+from evalscope.utils.logger import get_logger
+from .benchmark_util import Metrics
+from .db_util import PercentileMetrics
+
+logger = get_logger()
+
+
+def analyze_results(all_results):
+    """Analyze all test results and generate a summary report"""
+    summary = []
+    total_tokens = 0
+    total_time = 0
+
+    for result in all_results:
+        total_metrics = result[0]
+        percentile_metrics = result[1]
+        percentiles = percentile_metrics[PercentileMetrics.PERCENTILES]
+        try:
+            concurrency = total_metrics.get(Metrics.NUMBER_OF_CONCURRENCY, 0)
+            rps = total_metrics.get(Metrics.REQUEST_THROUGHPUT, 0)
+            avg_latency = total_metrics.get(Metrics.AVERAGE_LATENCY, 0)
+            p99_latency = percentile_metrics.get(PercentileMetrics.LATENCY)[percentiles.index('99%')]
+            avg_tps = total_metrics.get(Metrics.OUTPUT_TOKEN_THROUGHPUT, 0)
+            avg_ttft = total_metrics.get(Metrics.AVERAGE_TIME_TO_FIRST_TOKEN, 0)
+            p99_ttft = percentile_metrics.get(PercentileMetrics.TTFT)[percentiles.index('99%')]
+            success_rate = (total_metrics.get(Metrics.SUCCEED_REQUESTS, 0)
+                            / total_metrics.get(Metrics.TOTAL_REQUESTS, 1)) * 100
+            avg_tpot = total_metrics.get(Metrics.AVERAGE_TIME_PER_OUTPUT_TOKEN, 0)
+            p99_tpot = percentile_metrics.get(PercentileMetrics.TPOT)[percentiles.index('99%')]
+
+            # Ensure all values are valid numbers
+            if any(x is None for x in [concurrency, rps, avg_latency, p99_latency, avg_tps, avg_ttft]):
+                logger.warning(f'Warning: Test results for concurrency {concurrency} contain invalid data, skipped')
+                continue
+
+            summary.append([
+                concurrency,
+                f'{rps:.2f}' if rps is not None else 'N/A',
+                f'{avg_latency:.3f}' if avg_latency is not None else 'N/A',
+                f'{p99_latency:.3f}' if p99_latency is not None else 'N/A',
+                f'{avg_tps:.2f}' if avg_tps is not None else 'N/A',
+                f'{avg_ttft:.3f}' if avg_ttft is not None else 'N/A',
+                f'{success_rate:.1f}%' if success_rate is not None else 'N/A',
+                f'{p99_ttft:.3f}' if p99_ttft is not None else 'N/A',
+                f'{avg_tpot:.3f}' if avg_tpot is not None else 'N/A',
+                f'{p99_tpot:.3f}' if p99_tpot is not None else 'N/A',
+            ])
+
+            total_tokens += total_metrics.get(Metrics.AVERAGE_OUTPUT_TOKENS_PER_REQUEST, 0) * total_metrics.get(
+                Metrics.SUCCEED_REQUESTS, 0)
+            total_time += total_metrics.get(Metrics.TIME_TAKEN_FOR_TESTS, 0)
+        except Exception as e:
+            logger.warning(
+                f"Warning: Error processing results for concurrency {result.get('concurrency', 'unknown')}: {str(e)}")
+            continue
+
+    if not summary:
+        logger.warning('Error: No valid test result data')
+        return [], 0, 0
+
+    return summary, total_tokens, total_time
+
+
+def print_summary(all_results, model_name):
+    """Print test results summary"""
+    summary, total_tokens, total_time = analyze_results(all_results)
+
+    if not summary:
+        logger.warning('No available test result data to display')
+        return
+
+    console = Console(width=100)  # Set fixed width
+
+    # Create title panel
+    title = Text('Performance Test Summary Report', style='bold')
+    console.print(Panel(title, width=60))
+
+    # Print basic information
+    basic_info = Table(show_header=False, width=60)
+    basic_info.add_column('Name', style='cyan', width=25)
+    basic_info.add_column('Value', style='green', width=35)
+
+    basic_info.add_row('Model', model_name)
+    basic_info.add_row('Total Generated', f'{total_tokens:,} tokens')
+    basic_info.add_row('Total Test Time', f'{total_time:.2f} seconds')
+    basic_info.add_row('Avg Output Rate', f'{total_tokens / total_time:.2f} tokens/sec')
+
+    console.print('\nBasic Information:')
+    console.print(basic_info)
+
+    # Create detailed performance metrics table
+    table = Table(
+        title='Detailed Performance Metrics',
+        show_header=True,
+        header_style='bold cyan',
+        border_style='blue',
+        width=100,  # Set total table width
+        pad_edge=False,  # Reduce edge padding
+        min_width=60,  # Minimum width
+    )
+
+    # Add columns (set fixed column widths)
+    table.add_column('Conc.', justify='right', style='cyan')
+    table.add_column('RPS', justify='right')
+    table.add_column('Avg Lat.(s)', justify='right')
+    table.add_column('P99 Lat.(s)', justify='right')
+    table.add_column('Gen. toks/s', justify='right')
+    table.add_column('Avg TTFT(s)', justify='right')
+    table.add_column('P99 TTFT(s)', justify='right')
+    table.add_column('Avg TPOT(s)', justify='right')
+    table.add_column('P99 TPOT(s)', justify='right')
+    table.add_column('Success Rate', justify='right', style='green')
+
+    # Add data rows
+    for row in summary:
+        try:
+            # Set row style based on success rate
+            success_rate = float(row[6].rstrip('%'))
+            row_style = 'green' if success_rate >= 95 else 'yellow' if success_rate >= 80 else 'red'
+
+            table.add_row(
+                str(row[0]),  # Concurrency
+                f'{float(row[1]):.2f}',  # RPS
+                f'{float(row[2]):.3f}',  # Average Latency
+                f'{float(row[3]):.3f}',  # P99 Latency
+                f'{float(row[4]):.2f}',  # Average TPS
+                f'{float(row[5]):.3f}',  # First Token Latency
+                f'{float(row[7]):.3f}',  # P99 TTFT
+                f'{float(row[8]):.3f}',  # Average TPOT
+                f'{float(row[9]):.3f}',  # P99 TPOT
+                row[6],  # Success Rate
+                style=row_style)
+        except ValueError as e:
+            console.print(f'Warning: Error processing row data: {str(e)}', style='bold red')
+            continue
+
+    console.print('\n')
+    console.print(table)
+
+    # Calculate and display best performance configuration
+    try:
+        best_rps_idx = np.argmax([float(row[1]) if row[1] != 'N/A' else -1 for row in summary])
+        best_latency_idx = np.argmin([float(row[2]) if row[2] != 'N/A' else float('inf') for row in summary])
+
+        perf_info = Table(title='Best Performance Configuration', show_header=False, box=None, width=60)
+        perf_info.add_column('Metric', style='cyan', width=20)
+        perf_info.add_column('Value', style='green', width=40)
+
+        perf_info.add_row('Highest RPS', f'Concurrency {summary[best_rps_idx][0]} ({summary[best_rps_idx][1]} req/sec)')
+        perf_info.add_row('Lowest Latency',
+                          f'Concurrency {summary[best_latency_idx][0]} ({summary[best_latency_idx][2]} seconds)')
+
+        console.print('\n')
+        console.print(perf_info)
+
+        # Performance recommendations
+        recommendations = []
+        if best_rps_idx == len(summary) - 1:
+            recommendations.append(
+                'The system seems not to have reached its performance bottleneck, try higher concurrency')
+        elif best_rps_idx == 0:
+            recommendations.append('Consider lowering concurrency, current load may be too high')
+        else:
+            recommendations.append(f'Optimal concurrency range is around {summary[best_rps_idx][0]}')
+
+        success_rate = float(summary[-1][6][:-1])
+        if success_rate < 95:
+            recommendations.append(
+                'Success rate is low at high concurrency, check system resources or reduce concurrency')
+
+        recommend_text = Text('\nPerformance Recommendations:', style='bold cyan')
+        console.print(recommend_text)
+        for rec in recommendations:
+            console.print(f'• {rec}', style='yellow')
+
+    except Exception as e:
+        console.print(f'Warning: Error generating performance analysis: {str(e)}', style='bold red')

evalscope/report/__init__.py

@@ -1,6 +1,38 @@
 # Copyright (c) Alibaba, Inc. and its affiliates.
+from typing import TYPE_CHECKING
 
-from evalscope.report.app_arguments import add_argument
-from evalscope.report.combinator import gen_table, get_data_frame, get_report_list
-from evalscope.report.generator import ReportGenerator
-from evalscope.report.utils import Category, Report, ReportKey, Subset
+from evalscope.utils.import_utils import _LazyModule
+
+if TYPE_CHECKING:
+    from .combinator import gen_report_table, gen_table, get_data_frame, get_report_list
+    from .generator import ReportGenerator
+    from .utils import Category, Report, ReportKey, Subset
+
+else:
+    _import_structure = {
+        'combinator': [
+            'gen_table',
+            'get_data_frame',
+            'get_report_list',
+            'gen_report_table',
+        ],
+        'generator': [
+            'ReportGenerator',
+        ],
+        'utils': [
+            'Category',
+            'Report',
+            'ReportKey',
+            'Subset',
+        ],
+    }
+
+    import sys
+
+    sys.modules[__name__] = _LazyModule(
+        __name__,
+        globals()['__file__'],
+        _import_structure,
+        module_spec=__spec__,
+        extra_objects={},
+    )

evalscope/report/combinator.py

@@ -48,6 +48,14 @@ def gen_table(reports_path_list: list) -> str:
     return tabulate(table, headers=table.columns, tablefmt='grid', showindex=False)
 
 
+def gen_report_table(report: Report) -> str:
+    """
+    Generate a report table for a single report.
+    """
+    table = report.to_dataframe(flatten_metrics=True, flatten_categories=True)
+    return tabulate(table, headers=table.columns, tablefmt='grid', showindex=False)
+
+
 class ReportsRecorder:
     COMMON_DATASET_PATH = []
     CUSTOM_DATASET_PATH = []

evalscope/report/generator.py

@@ -1,24 +1,42 @@
 import pandas as pd
 from pandas import DataFrame
+from typing import TYPE_CHECKING
 
 from evalscope.constants import DataCollection
 from evalscope.report.utils import *
 
+if TYPE_CHECKING:
+    from evalscope.benchmarks import DataAdapter
+
 
 class ReportGenerator:
 
     @staticmethod
-    def gen_report(subset_score_map: dict, report_name: str, **kwargs) -> Report:
+    def gen_report(subset_score_map: dict, model_name: str, data_adapter: 'DataAdapter', **kwargs) -> Report:
         """
-        Generate report for specific dataset.
-        subset_score_map: e.g. {subset_name: [{'metric_name': 'AverageAccuracy', 'score': 0.3389, 'num': 100}, {'metric_name': 'WeightedAverageAccuracy', 'score': 0.3389, 'num': 100}]}
-        category_map: e.g. {'subset_name': ['category_name1', 'category_name2'], ...}
-        metric_list: e.g. [{'object': AverageAccuracy, 'name': 'AverageAccuracy'}, {'object': 'WeightedAverageAccuracy', 'name': 'WeightedAverageAccuracy'}]
+        Generate a report for a specific dataset based on provided subset scores.
+
+        Args:
+            subset_score_map (dict): A mapping from subset names to a list of score dictionaries.
+                    {
+                        'subset_name': [
+                            {'metric_name': 'AverageAccuracy', 'score': 0.3389, 'num': 100},
+                            {'metric_name': 'WeightedAverageAccuracy', 'score': 0.3389, 'num': 100}
+                        ],
+                        ...
+                    }
+            report_name (str): The name of the report to generate.
+            data_adapter (DataAdapter): An adapter object for data handling.
+
+        Returns:
+            Report: A structured report object containing metrics, categories, and subsets.
+
+            >>> report = gen_report(subset_score_map, "My Report", data_adapter, dataset_name="Dataset", model_name="Model")
         """  # noqa: E501
 
-        dataset_name = kwargs.get('dataset_name', None)
-        model_name = kwargs.get('model_name', None)
-        category_map = kwargs.get('category_map', {})
+        dataset_name = data_adapter.name
+        category_map = data_adapter.category_map
+        report_name = f'{model_name}@{dataset_name}'
 
         def flatten_subset() -> DataFrame:
             """
@@ -59,7 +77,13 @@ class ReportGenerator:
 
             metrics_list.append(Metric(name=metric_name, categories=categories))
 
-        report = Report(name=report_name, metrics=metrics_list, dataset_name=dataset_name, model_name=model_name)
+        report = Report(
+            name=report_name,
+            metrics=metrics_list,
+            dataset_name=dataset_name,
+            model_name=model_name,
+            dataset_description=data_adapter.description,
+            dataset_pretty_name=data_adapter.pretty_name)
         return report
 
     @staticmethod

evalscope/report/utils.py

@@ -1,4 +1,5 @@
 import json
+import os
 import pandas as pd
 from collections import defaultdict
 from dataclasses import asdict, dataclass, field
@@ -6,6 +7,9 @@ from typing import Any, Dict, List
 
 from evalscope.metrics import macro_mean, micro_mean
 from evalscope.utils import normalize_score
+from evalscope.utils.logger import get_logger
+
+logger = get_logger()
 
 
 @dataclass
@@ -70,13 +74,28 @@ class ReportKey:
     score = 'Score'
 
 
+ANALYSIS_PROMPT = """根据给出的json格式的模型评测结果，输出分析报告，要求如下：
+1. 报告分为 总体表现、关键指标分析、改进建议、结论 四部分
+2. 若模型有多种指标，将其分为低分、中分、高分三个部分，并列出markdown表格
+3. 只列出报告本身，不要有其他多余内容
+4. 输出报告语言为{language}
+
+```json
+{report_str}
+```
+"""
+
+
 @dataclass
 class Report:
     name: str = 'default_report'
     dataset_name: str = 'default_dataset'
+    dataset_pretty_name: str = ''
+    dataset_description: str = ''
     model_name: str = 'default_model'
     score: float = 0.0
     metrics: List[Metric] = field(default_factory=list)
+    analysis: str = 'N/A'
 
     def __post_init__(self):
         self.score = self.metrics[0].score  # NOTE: only use the first metric by default
@@ -84,19 +103,33 @@ class Report:
     def to_dict(self) -> Dict[str, Any]:
         return asdict(self)
 
+    def to_json_str(self) -> str:
+        return json.dumps(self.to_dict(), indent=4, ensure_ascii=False)
+
+    def to_json(self, json_file: str):
+        # ensure the directory exists
+        os.makedirs(os.path.dirname(json_file), exist_ok=True)
+        # write the report to a json file
+        with open(json_file, 'w', encoding='utf-8') as f:
+            json.dump(self.to_dict(), f, indent=4, ensure_ascii=False)
+
     @classmethod
     def from_dict(cls, data: dict):
         metrics = [Metric.from_dict(metric) for metric in data.get('metrics', [])]
         return cls(
             name=data['name'],
+            dataset_name=data['dataset_name'],
+            dataset_pretty_name=data.get('dataset_pretty_name'),
+            dataset_description=data.get('dataset_description'),
             score=data['score'],
+            model_name=data['model_name'],
             metrics=metrics,
-            dataset_name=data['dataset_name'],
-            model_name=data['model_name'])
+            analysis=data.get('analysis', 'N/A'),
+        )
 
     @classmethod
     def from_json(cls, json_file: str):
-        with open(json_file, 'r') as f:
+        with open(json_file, 'r', encoding='utf-8') as f:
             data = json.load(f)
         return cls.from_dict(data)
 
@@ -111,7 +144,7 @@ class Report:
                     table[ReportKey.category_name].append(category.name)
                     table[ReportKey.subset_name].append(subset.name)
                     table[ReportKey.num].append(subset.num)
-                    table[ReportKey.score].append(subset.score)  # TODO: convert to percentage
+                    table[ReportKey.score].append(subset.score)
             # NOTE: only flatten metrics if needed, use the first metric by default
             if not flatten_metrics:
                 break
@@ -131,3 +164,27 @@ class Report:
 
         df_categories.drop(columns=[ReportKey.category_name], inplace=True)
         return df_categories
+
+    def generate_analysis(self, judge_llm_config: dict) -> str:
+        import locale
+
+        from evalscope.metrics import LLMJudge
+
+        try:
+            # get the default locale
+            lang, _ = locale.getlocale()
+
+            if lang is None:
+                language = '中文'
+            else:
+                language = 'en' if lang.startswith('en') else '中文'
+
+            prompt = ANALYSIS_PROMPT.format(language=language, report_str=self.to_json_str())
+            judge_llm = LLMJudge(**judge_llm_config)
+            response = judge_llm(prompt)
+        except Exception as e:
+            logger.error(f'Error generating analysis: {e}')
+            response = 'N/A'
+
+        self.analysis = response
+        return response

evalscope/run.py

@@ -43,6 +43,9 @@ def run_single_task(task_cfg: TaskConfig, run_time: str) -> dict:
     else:
         result = evaluate_model(task_cfg, outputs)
 
+        logger.info(f'Finished evaluation for {task_cfg.model_id} on {task_cfg.datasets}')
+        logger.info(f'Output directory: {outputs.outputs_dir}')
+
     return result
 
 
@@ -109,6 +112,7 @@ def get_backend_manager_class(eval_backend: EvalBackend):
 def evaluate_model(task_cfg: TaskConfig, outputs: OutputsStructure) -> dict:
     """Evaluate the model based on the provided task configuration."""
     from evalscope.models import get_local_model
+    from evalscope.report import gen_table
 
     # Initialize evaluator
     eval_results = {}
@@ -122,10 +126,18 @@ def evaluate_model(task_cfg: TaskConfig, outputs: OutputsStructure) -> dict:
     task_cfg.dump_yaml(outputs.configs_dir)
     logger.info(task_cfg)
 
+    # Run evaluation for each evaluator
     for evaluator in evaluators:
         res_dict = evaluator.eval()
         eval_results[evaluator.dataset_name] = res_dict
 
+    # Make overall report
+    try:
+        report_table: str = gen_table([outputs.reports_dir])
+        logger.info(f'Overall report table: \n{report_table} \n')
+    except Exception:
+        logger.error('Failed to generate report table.')
+
     # Clean up
     if base_model is not None:
         import gc

evalscope/third_party/toolbench_static/toolbench_static.py

@@ -6,11 +6,12 @@ from typing import Union
 from evalscope.third_party.toolbench_static.eval import EvalArgs, run_eval
 from evalscope.third_party.toolbench_static.infer import InferArgs, run_infer
 from evalscope.utils import get_logger
+from evalscope.utils.deprecation_utils import deprecated
 from evalscope.utils.io_utils import json_to_dict, yaml_to_dict
 
 logger = get_logger()
 
-
+@deprecated(since='0.15.1', remove_in='0.18.0', alternative='Native implementation of ToolBench')
 def run_task(task_cfg: Union[str, dict]):
 
     if isinstance(task_cfg, str):

evalscope/utils/deprecation_utils.py

@@ -0,0 +1,42 @@
+import functools
+import inspect
+from typing import Callable, Optional
+
+from .logger import get_logger
+
+logger = get_logger()
+
+
+def deprecated(since: str, remove_in: Optional[str] = None, alternative: Optional[str] = None) -> Callable:
+    """
+    Decorator to mark functions as deprecated.
+
+    :param since: String indicating the version since deprecation
+    :param remove_in: Optional string indicating the version when it will be removed
+    :param alternative: Optional string suggesting an alternative
+    :return: Decorated function
+    """
+
+    def decorator(func: Callable) -> Callable:
+
+        @functools.wraps(func)
+        def wrapper(*args, **kwargs):
+            # Get the file name where the function is defined
+            file_name = inspect.getfile(func)
+
+            # Construct the warning message
+            warning_parts = [
+                f'{func.__name__} in {file_name} has been deprecated since version {since}',
+                f'and will be removed in version {remove_in}' if remove_in else None,
+                f'Use {alternative} instead' if alternative else None
+            ]
+            warning_message = '. '.join(filter(None, warning_parts))
+
+            # Log the warning
+            logger.warning(warning_message)
+
+            return func(*args, **kwargs)
+
+        return wrapper
+
+    return decorator

evalscope/utils/logger.py

@@ -10,7 +10,7 @@ simple_format = '%(asctime)s - %(name)s - %(levelname)s - %(message)s'
 
 detailed_formatter = logging.Formatter(detailed_format)
 simple_formatter = logging.Formatter(simple_format)
-DEFAULT_LEVEL = logging.DEBUG if os.getenv('LOG_LEVEL', 'INFO') == 'DEBUG' else logging.INFO
+DEFAULT_LEVEL = logging.DEBUG if os.getenv('EVALSCOPE_LOG_LEVEL', 'INFO') == 'DEBUG' else logging.INFO
 
 logging.basicConfig(format=simple_format, level=DEFAULT_LEVEL, force=True)
 

evalscope/utils/utils.py

@@ -10,6 +10,7 @@ import os
 import random
 import re
 import torch
+from inspect import signature
 from typing import Any, Dict, List, Tuple, Union
 
 from evalscope.utils.logger import get_logger
@@ -313,6 +314,17 @@ def seed_everything(seed: int):
         torch.backends.cudnn.deterministic = True
         torch.backends.cudnn.benchmark = False
 
+def get_supported_params(func):
+    """Get the supported parameters of a function."""
+    sig = signature(func)
+    return list(sig.parameters.keys())
+
+def parse_int_or_float(num):
+    number = float(num)
+    if number.is_integer():
+        return int(number)
+    return number
+
 if __name__ == '__main__':
     options = ['A', 'B', 'C', 'D']
     answers = ['Context .... ANSWER: A', 'answer: A']

evalscope/version.py

@@ -1,4 +1,4 @@
 # Copyright (c) Alibaba, Inc. and its affiliates.
 
-__version__ = '0.15.1'
-__release_datetime__ = '2025-04-30 12:00:00'
+__version__ = '0.16.1'
+__release_datetime__ = '2025-06-03 20:00:00'