evalscope 0.9.0__py3-none-any.whl → 0.10.0__py3-none-any.whl

evalscope/report/generator.py

@@ -0,0 +1,80 @@
+import pandas as pd
+from pandas import DataFrame
+
+from evalscope.constants import DataCollection
+from evalscope.report.utils import *
+
+
+class ReportGenerator:
+
+    @staticmethod
+    def gen_report(subset_score_map: dict, report_name: str, **kwargs) -> Report:
+        """
+        Generate report for specific dataset.
+        subset_score_map: e.g. {subset_name: [{'metric_name': 'AverageAccuracy', 'score': 0.3389, 'num': 100}, {'metric_name': 'WeightedAverageAccuracy', 'score': 0.3389, 'num': 100}]}
+        category_map: e.g. {'subset_name': ['category_name1', 'category_name2'], ...}
+        metric_list: e.g. [{'object': AverageAccuracy, 'name': 'AverageAccuracy'}, {'object': 'WeightedAverageAccuracy', 'name': 'WeightedAverageAccuracy'}]
+        """  # noqa: E501
+
+        dataset_name = kwargs.get('dataset_name', None)
+        model_name = kwargs.get('model_name', None)
+        category_map = kwargs.get('category_map', {})
+
+        def flatten_subset() -> DataFrame:
+            """
+            Flatten subset score map to a DataFrame.
+
+            Example:
+                        name  score  num   categories      metric_name
+            0       ARC-Easy    0.5    2    [default]  AverageAccuracy
+            1  ARC-Challenge    0.5    2    [default]  AverageAccuracy
+            """
+            subsets = []
+            for subset_name, scores in subset_score_map.items():
+                for score_item in scores:
+                    categories = category_map.get(subset_name, ['default'])
+                    if isinstance(categories, str):
+                        categories = [categories]
+                    subsets.append(
+                        dict(
+                            name=subset_name,
+                            score=score_item['score'],
+                            num=score_item['num'],
+                            metric_name=score_item['metric_name'],
+                            categories=tuple(categories)))
+            df = pd.DataFrame(subsets)
+            return df
+
+        df = flatten_subset()
+
+        metrics_list = []
+        for metric_name, group_metric in df.groupby('metric_name'):
+            categories = []
+            for category_name, group_category in group_metric.groupby('categories'):
+                subsets = []
+                for _, row in group_category.iterrows():
+                    subsets.append(Subset(name=row['name'], score=row['score'], num=row['num']))
+
+                categories.append(Category(name=category_name, subsets=subsets))
+
+            metrics_list.append(Metric(name=metric_name, categories=categories))
+
+        report = Report(name=report_name, metrics=metrics_list, dataset_name=dataset_name, model_name=model_name)
+        return report
+
+    @staticmethod
+    def gen_collection_report(df: DataFrame, all_dataset_name: str, model_name: str) -> Report:
+        categories = []
+        for category_name, group_category in df.groupby('categories'):
+            subsets = []
+            for (dataset_name, subset_name), group_subset in group_category.groupby(['dataset_name', 'subset_name']):
+                avg_score = group_subset['score'].mean()
+                num = group_subset['score'].count()
+                subsets.append(Subset(name=f'{dataset_name}/{subset_name}', score=float(avg_score), num=int(num)))
+
+            categories.append(Category(name=category_name, subsets=subsets))
+        return Report(
+            name=DataCollection.NAME,
+            metrics=[Metric(name='Average', categories=categories)],
+            dataset_name=all_dataset_name,
+            model_name=model_name)

evalscope/report/utils.py

@@ -0,0 +1,133 @@
+import json
+import pandas as pd
+from collections import defaultdict
+from dataclasses import asdict, dataclass, field
+from typing import Any, Dict, List
+
+from evalscope.metrics import macro_mean, micro_mean
+from evalscope.utils import normalize_score
+
+
+@dataclass
+class Subset:
+    name: str = 'default_subset'
+    score: float = 0.0
+    num: int = 0
+
+    def __post_init__(self):
+        self.score = normalize_score(self.score)
+
+
+@dataclass
+class Category:
+    name: tuple[str] = field(default_factory=tuple)
+    num: int = 0
+    score: float = 0.0
+    macro_score: float = 0.0
+    subsets: List[Subset] = field(default_factory=list)
+
+    def __post_init__(self):
+        if isinstance(self.name, str):
+            # ensure name is tuple format
+            self.name = (self.name, )
+        self.num = sum(subset.num for subset in self.subsets)
+        self.score = normalize_score(micro_mean(self.subsets))
+        self.macro_score = normalize_score(macro_mean(self.subsets))
+
+    @classmethod
+    def from_dict(cls, data: dict):
+        subsets = [Subset(**subset) for subset in data.get('subsets', [])]
+        return cls(name=data['name'], subsets=subsets)
+
+
+@dataclass
+class Metric:
+    name: str = 'default_metric'
+    num: int = 0
+    score: float = 0.0
+    macro_score: float = 0.0
+    categories: List[Category] = field(default_factory=list)
+
+    def __post_init__(self):
+        self.num = sum(category.num for category in self.categories)
+        self.score = normalize_score(micro_mean(self.categories))
+        self.macro_score = normalize_score(macro_mean(self.categories))
+
+    @classmethod
+    def from_dict(cls, data: dict):
+        categories = [Category.from_dict(category) for category in data.get('categories', [])]
+        return cls(name=data['name'], categories=categories)
+
+
+class ReportKey:
+    model_name = 'Model'
+    dataset_name = 'Dataset'
+    metric_name = 'Metric'
+    category_name = 'Category'
+    category_prefix = 'Cat.'
+    subset_name = 'Subset'
+    num = 'Num'
+    score = 'Score'
+
+
+@dataclass
+class Report:
+    name: str = 'default_report'
+    dataset_name: str = 'default_dataset'
+    model_name: str = 'default_model'
+    score: float = 0.0
+    metrics: List[Metric] = field(default_factory=list)
+
+    def __post_init__(self):
+        self.score = self.metrics[0].score  # NOTE: only use the first metric by default
+
+    def to_dict(self) -> Dict[str, Any]:
+        return asdict(self)
+
+    @classmethod
+    def from_dict(cls, data: dict):
+        metrics = [Metric.from_dict(metric) for metric in data.get('metrics', [])]
+        return cls(
+            name=data['name'],
+            score=data['score'],
+            metrics=metrics,
+            dataset_name=data['dataset_name'],
+            model_name=data['model_name'])
+
+    @classmethod
+    def from_json(cls, json_file: str):
+        with open(json_file, 'r') as f:
+            data = json.load(f)
+        return cls.from_dict(data)
+
+    def to_dataframe(self, flatten_metrics: bool = True, flatten_categories: bool = True):
+        table = defaultdict(list)
+        for metric in self.metrics:
+            for category in metric.categories:
+                for subset in category.subsets:
+                    table[ReportKey.model_name].append(self.model_name)
+                    table[ReportKey.dataset_name].append(self.dataset_name)
+                    table[ReportKey.metric_name].append(metric.name)
+                    table[ReportKey.category_name].append(category.name)
+                    table[ReportKey.subset_name].append(subset.name)
+                    table[ReportKey.num].append(subset.num)
+                    table[ReportKey.score].append(subset.score)  # TODO: convert to percentage
+            # NOTE: only flatten metrics if needed, use the first metric by default
+            if not flatten_metrics:
+                break
+        df = pd.DataFrame.from_dict(table, orient='columns')
+        if flatten_categories:
+            df = self._flatten_categories(df)
+        return df
+
+    def _flatten_categories(self, df: pd.DataFrame):
+        # expand categories to multiple rows
+        df_categories = df.copy()
+        # multi-level aggregation for categories
+        max_depth = df_categories[ReportKey.category_name].apply(len).max()
+        for level in range(max_depth):
+            df_categories[f'{ReportKey.category_prefix}{level}'] = df_categories[ReportKey.category_name].apply(
+                lambda x: x[level] if len(x) > level else None)
+
+        df_categories.drop(columns=[ReportKey.category_name], inplace=True)
+        return df_categories

evalscope/run.py

@@ -5,18 +5,17 @@ Run evaluation for LLMs.
 import os.path
 from argparse import Namespace
 from datetime import datetime
-from typing import List, Optional, Union
+from typing import TYPE_CHECKING, List, Optional, Union
 
-from evalscope.arguments import parse_args
-from evalscope.benchmarks import Benchmark, BenchmarkMeta
 from evalscope.config import TaskConfig, parse_task_config
-from evalscope.constants import DEFAULT_WORK_DIR, EvalBackend
-from evalscope.evaluator import Evaluator
-from evalscope.models import LocalModel, get_local_model, initialize_model_adapter
+from evalscope.constants import DataCollection, EvalBackend
 from evalscope.utils import seed_everything
-from evalscope.utils.io_utils import OutputsStructure, are_paths_same
+from evalscope.utils.io_utils import OutputsStructure
 from evalscope.utils.logger import configure_logging, get_logger
 
+if TYPE_CHECKING:
+    from evalscope.models import LocalModel
+
 logger = get_logger()
 
 
@@ -50,8 +49,8 @@ def setup_work_directory(task_cfg: TaskConfig, run_time: str):
     if task_cfg.use_cache:
         task_cfg.work_dir = task_cfg.use_cache
         logger.info(f'Set resume from {task_cfg.work_dir}')
-    elif are_paths_same(task_cfg.work_dir, DEFAULT_WORK_DIR):
-        task_cfg.work_dir = os.path.join(task_cfg.work_dir, run_time)
+    # elif are_paths_same(task_cfg.work_dir, DEFAULT_WORK_DIR):
+    task_cfg.work_dir = os.path.join(task_cfg.work_dir, run_time)
 
     outputs = OutputsStructure(outputs_dir=task_cfg.work_dir)
 
@@ -98,6 +97,8 @@ def get_backend_manager_class(eval_backend: EvalBackend):
 
 def evaluate_model(task_cfg: TaskConfig, outputs: OutputsStructure) -> dict:
     """Evaluate the model based on the provided task configuration."""
+    from evalscope.models import get_local_model
+
     # Initialize evaluator
     eval_results = {}
     base_model = get_local_model(task_cfg)
@@ -117,10 +118,13 @@ def evaluate_model(task_cfg: TaskConfig, outputs: OutputsStructure) -> dict:
     return eval_results
 
 
-def create_evaluator(task_cfg: TaskConfig, dataset_name: str, outputs: OutputsStructure, base_model: LocalModel):
+def create_evaluator(task_cfg: TaskConfig, dataset_name: str, outputs: OutputsStructure, base_model: 'LocalModel'):
     """Create an evaluator object for the specified dataset."""
+    from evalscope.benchmarks import Benchmark, BenchmarkMeta
+    from evalscope.evaluator import Evaluator
+    from evalscope.models import initialize_model_adapter
 
-    if dataset_name == 'data_collection':
+    if dataset_name == DataCollection.NAME:
         # EvaluatorCollection is a collection of evaluators
         from evalscope.collections import EvaluatorCollection
         return EvaluatorCollection(task_cfg, outputs)
@@ -143,6 +147,7 @@ def create_evaluator(task_cfg: TaskConfig, dataset_name: str, outputs: OutputsSt
 
 
 def main():
+    from evalscope.arguments import parse_args
     args = parse_args()
     run_task(args)
 

evalscope/summarizer.py

@@ -6,7 +6,7 @@ from typing import List, Union
 
 from evalscope.config import TaskConfig, parse_task_config
 from evalscope.constants import EvalBackend
-from evalscope.tools.combine_reports import gen_table
+from evalscope.report import gen_table
 from evalscope.utils import csv_to_list, get_latest_folder_path
 from evalscope.utils.io_utils import OutputsStructure, json_to_dict, yaml_to_dict
 from evalscope.utils.logger import get_logger

evalscope/utils/chat_service.py

@@ -5,7 +5,6 @@ from contextlib import contextmanager
 from functools import partial
 from pydantic import BaseModel, Field
 from threading import Thread
-from transformers import TextIteratorStreamer
 from typing import Any, List, Literal, Optional, Union
 
 
@@ -96,6 +95,7 @@ class ChatService:
 
     def __init__(self, model_path, attn_implementation):
         from modelscope import AutoModelForCausalLM, AutoTokenizer
+        from transformers import TextIteratorStreamer
 
         self.tokenizer = AutoTokenizer.from_pretrained(model_path, trust_remote_code=True)
         self.model = AutoModelForCausalLM.from_pretrained(

evalscope/utils/logger.py

@@ -17,6 +17,7 @@ logging.basicConfig(format=simple_format, level=DEFAULT_LEVEL)
 # disable datasets logging
 logging.getLogger('datasets').setLevel(logging.WARNING)
 logging.getLogger('modelscope').setLevel(logging.WARNING)
+logging.getLogger('httpx').setLevel(logging.WARNING)
 
 
 def get_logger(log_file: Optional[str] = None, log_level: int = DEFAULT_LEVEL, file_mode: str = 'w', force=False):

evalscope/utils/model_utils.py

@@ -1,5 +1,8 @@
 from enum import Enum
-from transformers import GenerationConfig
+from typing import TYPE_CHECKING
+
+if TYPE_CHECKING:
+    from transformers import GenerationConfig
 
 
 class EvalBackend(Enum):
@@ -11,7 +14,7 @@ class EvalBackend(Enum):
     THIRD_PARTY = 'ThirdParty'
 
 
-def fix_do_sample_warning(generation_config: GenerationConfig) -> None:
+def fix_do_sample_warning(generation_config: 'GenerationConfig') -> None:
     # Use the default values of temperature/top_p/top_k in generation_config.
     if generation_config.temperature == 0:
         generation_config.do_sample = False

evalscope/version.py

@@ -1,4 +1,4 @@
 # Copyright (c) Alibaba, Inc. and its affiliates.
 
-__version__ = '0.9.0'
-__release_datetime__ = '2025-01-03 18:00:00'
+__version__ = '0.10.0'
+__release_datetime__ = '2025-01-20 20:00:00'

{evalscope-0.9.0.dist-info → evalscope-0.10.0.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: evalscope
-Version: 0.9.0
+Version: 0.10.0
 Summary: EvalScope: Lightweight LLMs Evaluation Framework
 Home-page: https://github.com/modelscope/evalscope
 Author: ModelScope team
@@ -92,6 +92,11 @@ Requires-Dist: numpy; extra == "all"
 Requires-Dist: sse-starlette; extra == "all"
 Requires-Dist: transformers; extra == "all"
 Requires-Dist: unicorn; extra == "all"
+Requires-Dist: gradio>=5.4.0; extra == "all"
+Requires-Dist: plotly>=5.23.0; extra == "all"
+Provides-Extra: app
+Requires-Dist: gradio>=5.4.0; extra == "app"
+Requires-Dist: plotly>=5.23.0; extra == "app"
 Provides-Extra: inner
 Requires-Dist: absl-py; extra == "inner"
 Requires-Dist: accelerate; extra == "inner"
@@ -210,6 +215,8 @@ Please scan the QR code below to join our community groups:
 
 
 ## 🎉 News
+- 🔥 **[2025.01.20]** Support for visualizing evaluation results, including single model evaluation results and multi-model comparison, refer to the [📖 Visualizing Evaluation Results](https://evalscope.readthedocs.io/en/latest/get_started/visulization.html) for more details; Added [`iquiz`](https://modelscope.cn/datasets/AI-ModelScope/IQuiz/summary) evaluation example, evaluating the IQ and EQ of the model.
+- 🔥 **[2025.01.07]** Native backend: Support for model API evaluation is now available. Refer to the [📖 Model API Evaluation Guide](https://evalscope.readthedocs.io/en/latest/get_started/basic_usage.html#api) for more details. Additionally, support for the `ifeval` evaluation benchmark has been added.
 - 🔥🔥 **[2024.12.31]** Support for adding benchmark evaluations, refer to the [📖 Benchmark Evaluation Addition Guide](https://evalscope.readthedocs.io/en/latest/advanced_guides/add_benchmark.html); support for custom mixed dataset evaluations, allowing for more comprehensive model evaluations with less data, refer to the [📖 Mixed Dataset Evaluation Guide](https://evalscope.readthedocs.io/en/latest/advanced_guides/collection/index.html).
 - 🔥 **[2024.12.13]** Model evaluation optimization: no need to pass the `--template-type` parameter anymore; supports starting evaluation with `evalscope eval --args`. Refer to the [📖 User Guide](https://evalscope.readthedocs.io/en/latest/get_started/basic_usage.html) for more details.
 - 🔥 **[2024.11.26]** The model inference service performance evaluator has been completely refactored: it now supports local inference service startup and Speed Benchmark; asynchronous call error handling has been optimized. For more details, refer to the [📖 User Guide](https://evalscope.readthedocs.io/en/latest/user_guides/stress_test/index.html).
@@ -374,15 +381,85 @@ run_task(task_cfg="config.json")
 - `--limit`: Maximum amount of evaluation data for each dataset. If not specified, it defaults to evaluating all data. Can be used for quick validation
 
 ### Output Results
+```text
++-----------------------+----------------+-----------------+-----------------+---------------+-------+---------+
+| Model Name            | Dataset Name   | Metric Name     | Category Name   | Subset Name   |   Num |   Score |
++=======================+================+=================+=================+===============+=======+=========+
+| Qwen2.5-0.5B-Instruct | gsm8k          | AverageAccuracy | default         | main          |     5 |     0.4 |
++-----------------------+----------------+-----------------+-----------------+---------------+-------+---------+
+| Qwen2.5-0.5B-Instruct | ai2_arc        | AverageAccuracy | default         | ARC-Easy      |     5 |     0.8 |
++-----------------------+----------------+-----------------+-----------------+---------------+-------+---------+
+| Qwen2.5-0.5B-Instruct | ai2_arc        | AverageAccuracy | default         | ARC-Challenge |     5 |     0.4 |
++-----------------------+----------------+-----------------+-----------------+---------------+-------+---------+
+```
+
+## 📈 Visualization of Evaluation Results
+
+1. Install the dependencies required for visualization, including gradio, plotly, etc.
+```bash
+pip install 'evalscope[app]'
 ```
-+-----------------------+-------------------+-----------------+
-| Model                 | ai2_arc           | gsm8k           |
-+=======================+===================+=================+
-| Qwen2.5-0.5B-Instruct | (ai2_arc/acc) 0.6 | (gsm8k/acc) 0.6 |
-+-----------------------+-------------------+-----------------+
+
+2. Start the Visualization Service
+
+Run the following command to start the visualization service.
+```bash
+evalscope app
+```
+You can access the visualization service in the browser if the following output appears.
+```text
+* Running on local URL:  http://127.0.0.1:7861
+
+To create a public link, set `share=True` in `launch()`.
 ```
 
-## ⚙️ Complex Evaluation
+<table>
+  <tr>
+    <td style="text-align: center;">
+      <img src="docs/zh/get_started/images/setting.png" alt="Setting" style="width: 100%;" />
+      <p>Setting Interface</p>
+    </td>
+    <td style="text-align: center;">
+      <img src="docs/zh/get_started/images/model_compare.png" alt="Model Compare" style="width: 100%;" />
+      <p>Model Comparison</p>
+    </td>
+  </tr>
+  <tr>
+    <td style="text-align: center;">
+      <img src="docs/zh/get_started/images/report_overview.png" alt="Report Overview" style="width: 100%;" />
+      <p>Report Overview</p>
+    </td>
+    <td style="text-align: center;">
+      <img src="docs/zh/get_started/images/report_details.png" alt="Report Details" style="width: 100%;" />
+      <p>Report Details</p>
+    </td>
+  </tr>
+</table>
+
+For more details, refer to: [📖 Visualization of Evaluation Results](https://evalscope.readthedocs.io/en/latest/get_started/visulization.html)
+
+## 🌐 Evaluation of Specified Model API
+
+Specify the model API service address (api_url) and API Key (api_key) to evaluate the deployed model API service. In this case, the `eval-type` parameter must be specified as `service`, for example:
+
+For example, to launch a model service using [vLLM](https://github.com/vllm-project/vllm):
+
+```shell
+export VLLM_USE_MODELSCOPE=True && python -m vllm.entrypoints.openai.api_server --model Qwen/Qwen2.5-0.5B-Instruct --served-model-name qwen2.5 --trust_remote_code --port 8801
+```
+Then, you can use the following command to evaluate the model API service:
+```shell
+evalscope eval \
+ --model qwen2.5 \
+ --api-url http://127.0.0.1:8801/v1/chat/completions \
+ --api-key EMPTY \
+ --eval-type service \
+ --datasets gsm8k \
+ --limit 10
+```
+
+## ⚙️ Custom Parameter Evaluation
+
 For more customized evaluations, such as customizing model parameters or dataset parameters, you can use the following command. The evaluation startup method is the same as simple evaluation. Below shows how to start the evaluation using the `eval` command:
 
 ```shell