evalscope 0.16.0__py3-none-any.whl → 0.16.1__py3-none-any.whl

evalscope/evaluator/evaluator.py

@@ -13,7 +13,7 @@ from evalscope.benchmarks import DataAdapter
 from evalscope.config import TaskConfig
 from evalscope.constants import AnswerKeys, DumpMode, EvalStage, EvalType, JudgeStrategy, ReviewKeys
 from evalscope.models import BaseModelAdapter
-from evalscope.report import Report, gen_table
+from evalscope.report import Report, gen_report_table
 from evalscope.utils import dict_torch_dtype_to_str, gen_hash
 from evalscope.utils.io_utils import OutputsStructure, dump_jsonl_data, jsonl_to_list
 from evalscope.utils.logger import get_logger
@@ -46,7 +46,6 @@ class Evaluator(object):
         self.dataset_name = data_adapter.name
         self.dataset_name_or_path = os.path.expanduser(data_adapter.dataset_id)
         self.model_name = task_cfg.model_id
-        self.custom_task_name = f'{self.model_name}_{self.dataset_name}'
 
         self.data_adapter = data_adapter
         self.model_adapter = model_adapter
@@ -79,8 +78,16 @@ class Evaluator(object):
         # Limit and index prompts
         limited_prompts = defaultdict(list)
         for subset_name, prompts_list in prompts.items():
-            limit = self.task_cfg.limit or len(prompts_list)
-            for index, prompt in enumerate(prompts_list[:limit]):
+            # If limit is None, use all prompts
+            if self.task_cfg.limit is None:
+                limit = len(prompts_list)
+            else:
+                if isinstance(self.task_cfg.limit, int):
+                    limit = self.task_cfg.limit
+                elif isinstance(self.task_cfg.limit, float):
+                    limit = int(len(prompts_list) * self.task_cfg.limit)
+            # Limit the number of prompts
+            for index, prompt in enumerate(prompts_list[:min(limit, len(prompts_list))]):
                 prompt[AnswerKeys.INDEX] = index
                 limited_prompts[subset_name].append(prompt)
 
@@ -371,41 +378,45 @@ class Evaluator(object):
 
         return metric_score
 
-    def dump_report(self, reviews_score_all: List[dict], use_table: bool = True):
+    def dump_report(self, reviews_score_all: List[dict]):
         """
         Get report for total reviews of specific dataset.
         It is required to rewrite this method to support your own evaluator.
 
         Args:
             reviews_score_all: reviews score list. Generated by func self.data_adapter.compute_metric().
-            use_table: whether to generate table for reports. Default to True.
 
         Returns: None
         """
+        report_path = os.path.join(self.outputs_structure.reports_dir, self.model_name)
+        os.makedirs(report_path, exist_ok=True)
         # Get report map
         report_map: Report = self.data_adapter.gen_report(
-            subset_score_map=reviews_score_all,
-            report_name=self.custom_task_name,
-            model_name=self.model_name,
-            dataset_name=self.dataset_name)
-
-        # Dump report
-        report_path: str = os.path.join(self.outputs_structure.reports_dir, self.model_name,
-                                        self.dataset_name + '.json')
-        os.makedirs(os.path.dirname(report_path), exist_ok=True)
+            subset_score_map=reviews_score_all, model_name=self.model_name)
 
-        # Write report
-        with open(report_path, 'w', encoding='utf-8') as f:
-            f.write(json.dumps(report_map.to_dict(), ensure_ascii=False, indent=4))
-        logger.info(f'Dump report: {report_path} \n')
+        # Post process report
+        self.data_adapter.post_process_report(report_map, report_path=report_path)
 
         # Make table
-        if use_table:
-            try:
-                report_table: str = gen_table([self.outputs_structure.reports_dir])
-                logger.info(f'Report table: \n{report_table} \n')
-            except Exception:
-                logger.error('Failed to generate report table.')
+        try:
+            report_table = gen_report_table(report_map)
+            logger.info(f'{self.dataset_name_or_path} report table: \n{report_table} \n')
+        except Exception:
+            logger.error('Failed to generate report table.')
+
+        # Make report analysis
+        if self.task_cfg.analysis_report:
+            logger.info('Generating report analysis, please wait ...')
+            analysis = report_map.generate_analysis(self.task_cfg.judge_model_args)
+            logger.info('Report analysis:\n%s', analysis)
+        else:
+            logger.info('Skipping report analysis (`analysis_report=False`).')
+
+        # Dump report
+        report_file = os.path.join(report_path, f'{self.dataset_name}.json')
+        report_map.to_json(report_file)
+        logger.info(f'Dump report to: {report_file} \n')
+
         return report_map
 
     def eval(self, **kwargs) -> dict:
@@ -431,7 +442,7 @@ class Evaluator(object):
             stage == 'review': return the reviews_map
         """
 
-        logger.info(f'**** Start evaluating on dataset {self.dataset_name_or_path} ****')
+        logger.info(f'Start evaluating on dataset {self.dataset_name_or_path}')
 
         reviews_score_all = {}  # {subset_name: (score, num)}
         stage_answers_dict = {}
@@ -461,6 +472,6 @@ class Evaluator(object):
         # Generate report
         report_map = self.dump_report(reviews_score_all)
 
-        logger.info(f'**** Evaluation finished on {self.dataset_name_or_path} ****\n')
+        logger.info(f'Evaluation finished on {self.dataset_name_or_path}')
 
         return report_map

evalscope/metrics/__init__.py

@@ -9,7 +9,7 @@ if TYPE_CHECKING:
     from .metrics import (bleu_ngram_one_sample, exact_match, macro_mean, mean, micro_mean, simple_f1_score,
                           weighted_mean)
     from .named_metrics import Metric, metric_registry
-    from .rouge_metric import compute_rouge_score_one_sample_zh
+    from .rouge_metric import compute_rouge_score, compute_rouge_score_one_sample, compute_rouge_score_one_sample_zh
 
 else:
     _import_structure = {
@@ -28,6 +28,8 @@ else:
         ],
         'rouge_metric': [
             'compute_rouge_score_one_sample_zh',
+            'compute_rouge_score',
+            'compute_rouge_score_one_sample',
         ],
         'llm_judge': [
             'LLMJudge',

evalscope/metrics/bundled_rouge_score/rouge_scorer.py

@@ -88,11 +88,11 @@ class RougeScorer(scoring.BaseScorer):
     """
 
     def __init__(self, rouge_types, use_stemmer=False, split_summaries=False, tokenizer=None):
-        check_nltk_data()
         self.rouge_types = rouge_types
         if tokenizer:
             self._tokenizer = tokenizer
         else:
+            check_nltk_data()
             self._tokenizer = tokenizers.DefaultTokenizer(use_stemmer)
             logging.info('Using default tokenizer.')
 

evalscope/metrics/llm_judge.py

@@ -22,6 +22,9 @@ B: INCORRECT
 Just return the letters "A" or "B", with no text around it.
 """  # noqa: E501
 
+DEFAULT_JUDGE_MODEL = 'Qwen/Qwen3-235B-A22B'
+DEFAULT_API_URL = 'https://api-inference.modelscope.cn/v1/'
+
 
 class LLMJudge:
     """
@@ -47,12 +50,12 @@ class LLMJudge:
             prompt_template (str, optional): Prompt template for the judge
             generation_config (dict, optional): Generation configuration for the judge
         """
-        self.api_key = api_key or os.environ.get('OPENAI_API_KEY', 'EMPTY')
-        self.api_url = api_url or os.environ.get('OPENAI_API_BASE', 'https://api.openai.com/v1')
-        self.model_id = model_id or os.environ.get('LOCAL_LLM', 'gpt-4')
+        self.api_key = api_key or os.environ.get('MODELSCOPE_SDK_TOKEN', 'EMPTY')
+        self.api_url = api_url or os.environ.get('MODELSCOPE_API_BASE', DEFAULT_API_URL)
+        self.model_id = model_id or os.environ.get('MODELSCOPE_JUDGE_LLM', DEFAULT_JUDGE_MODEL)
         self.system_prompt = system_prompt or os.environ.get('JUDGE_SYSTEM_PROMPT', None)
         self.prompt_template = prompt_template or os.environ.get('JUDGE_PROMPT_TEMPLATE', DEFAULT_PROMPT_TEMPLATE)
-        self.generation_config = generation_config
+        self.generation_config = generation_config or {}
 
         from evalscope.models import ServerModelAdapter
 
@@ -74,6 +77,10 @@ class LLMJudge:
         if self.generation_config:
             infer_cfg.update(self.generation_config)
 
+        if self.model_id == DEFAULT_JUDGE_MODEL:
+            # Disable thinking for the default judge model
+            infer_cfg['enable_thinking'] = self.generation_config.get('enable_thinking', False)
+
         try:
             # Send request using ServerModelAdapter
             response = self.server_adapter.process_single_input(input_data, infer_cfg)
@@ -82,7 +89,7 @@ class LLMJudge:
             llm_response = response.get('choices', [{}])[0].get('message', {}).get('content', '')
             return llm_response
         except Exception as e:
-            logger.error(f'Error during LLM evaluation: {e}')
+            logger.error(f'Error occurred during {self.model_id}@{self.api_url} LLM judge evaluation: {e}')
             return ''
 
     def build_prompt(self, pred: str, gold: str, question: Optional[str] = None):

evalscope/metrics/math_parser.py

@@ -4,7 +4,7 @@ The logic in this file largely borrows from Qwen2.5-Math codebase at https://git
 # flake8: noqa
 import re
 import regex
-from latex2sympy2 import latex2sympy
+from latex2sympy2_extended import latex2sympy
 from math import isclose
 from sympy import N, simplify
 from sympy.parsing.latex import parse_latex

evalscope/models/adapters/server_adapter.py

@@ -1,11 +1,11 @@
 import openai
 from collections import defaultdict
-from inspect import signature
 from openai.types.chat import ChatCompletion, ChatCompletionChunk
 from openai.types.chat.chat_completion import ChatCompletionMessage, Choice
 from typing import List, Optional, Union
 
 from evalscope.utils.logger import get_logger
+from evalscope.utils.utils import get_supported_params
 from .base_adapter import BaseModelAdapter
 
 logger = get_logger()
@@ -31,7 +31,7 @@ class ServerModelAdapter(BaseModelAdapter):
             api_key=api_key,
             base_url=self.api_url,
         )
-        self.supported_params = self._get_supported_params()
+        self.supported_params = get_supported_params(self.client.chat.completions.create)
 
         self.seed = kwargs.get('seed', None)
         self.timeout = kwargs.get('timeout', 60)
@@ -39,10 +39,6 @@ class ServerModelAdapter(BaseModelAdapter):
         self.model_cfg = {'api_url': api_url, 'model_id': model_id, 'api_key': api_key}
         super().__init__(model=None, model_cfg=self.model_cfg, **kwargs)
 
-    def _get_supported_params(self):
-        sig = signature(self.client.chat.completions.create)
-        return list(sig.parameters.keys())
-
     def predict(self, inputs: List[dict], infer_cfg: Optional[dict] = None) -> List[dict]:
         """
         Model prediction func.

evalscope/perf/arguments.py

@@ -60,8 +60,8 @@ class Arguments:
     min_tokens: Optional[int] = None  # Minimum number of tokens in the response
     n_choices: Optional[int] = None  # Number of response choices
     seed: Optional[int] = 0  # Random seed for reproducibility
-    stop: Optional[List[str]] = field(default_factory=list)  # Stop sequences for the response
-    stop_token_ids: Optional[List[str]] = field(default_factory=list)  # Stop token IDs for the response
+    stop: Optional[List[str]] = None  # Stop sequences for the response
+    stop_token_ids: Optional[List[str]] = None  # Stop token IDs for the response
     stream: Optional[bool] = True  # Whether to stream the response
     temperature: float = 0.0  # Temperature setting for the response
     top_p: Optional[float] = None  # Top-p (nucleus) sampling setting for the response

evalscope/perf/benchmark.py

@@ -1,11 +1,8 @@
 import asyncio
-import copy
 import json
 import numpy as np
-import os
 import platform
 import sqlite3
-import threading
 import time
 from http import HTTPStatus
 from tqdm import tqdm
@@ -17,7 +14,6 @@ from evalscope.perf.plugin.registry import ApiRegistry, DatasetRegistry
 from evalscope.perf.utils.benchmark_util import BenchmarkData, BenchmarkMetrics
 from evalscope.perf.utils.db_util import create_result_table, get_result_db_path, insert_benchmark_data, summary_result
 from evalscope.perf.utils.handler import add_signal_handlers, exception_handler
-from evalscope.perf.utils.local_server import start_app
 from evalscope.utils.logger import get_logger
 
 logger = get_logger()
@@ -164,11 +160,6 @@ async def statistic_benchmark_metric(benchmark_data_queue: asyncio.Queue, args:
 
 @exception_handler
 async def connect_test(args: Arguments) -> bool:
-    if args.api.startswith('local'):
-        #  start local server
-        server = threading.Thread(target=start_app, args=(copy.deepcopy(args), ), daemon=True)
-        server.start()
-
     if (not args.no_test_connection) and (not await test_connection(args)):
         raise TimeoutError('Test connection failed')
 

evalscope/perf/main.py

@@ -2,9 +2,11 @@ import asyncio
 import copy
 import os
 import platform
+import threading
 import time
 from argparse import Namespace
 
+from evalscope.perf.utils.local_server import start_app
 from evalscope.perf.utils.log_utils import init_swanlab, init_wandb
 from evalscope.utils.logger import configure_logging, get_logger
 from evalscope.utils.utils import seed_everything
@@ -82,6 +84,11 @@ def run_perf_benchmark(args):
     if args.swanlab_api_key:
         init_swanlab(args)
 
+    # Initialize local server if needed
+    if args.api.startswith('local'):
+        #  start local server
+        server = threading.Thread(target=start_app, args=(copy.deepcopy(args), ), daemon=True)
+        server.start()
     # Start benchmark
     if len(args.number) == 1:
         return run_one_benchmark(args, output_path=output_path)

evalscope/perf/plugin/datasets/custom.py

@@ -22,3 +22,18 @@ class CustomDatasetPlugin(DatasetPluginBase):
                     yield [{'role': 'user', 'content': prompt}]
                 else:
                     yield prompt
+
+
+if __name__ == '__main__':
+    from evalscope.perf.arguments import Arguments
+    from evalscope.perf.main import run_perf_benchmark
+
+    args = Arguments(
+        model='qwen2.5-7b-instruct',
+        url='https://dashscope.aliyuncs.com/compatible-mode/v1/chat/completions',
+        dataset_path='outputs/perf_data.txt',
+        api_key='EMPTY',
+        dataset='custom',
+    )
+
+    run_perf_benchmark(args)

evalscope/perf/utils/benchmark_util.py

@@ -38,7 +38,7 @@ class BenchmarkData:
             self.first_chunk_latency = self.query_latency
             self.n_chunks = 1
             self.n_chunks_time = self.query_latency
-        self.time_per_output_token = self.n_chunks_time / self.completion_tokens
+        self.time_per_output_token = self.n_chunks_time / self.n_chunks
 
     def _calculate_tokens(self, api_plugin):
         self.prompt_tokens, self.completion_tokens = \

evalscope/perf/utils/local_server.py

@@ -96,6 +96,7 @@ def create_app(model, attn_implementation=None) -> FastAPI:
 
 
 def start_app(args: Arguments):
+    logger.info('Starting local server, please wait...')
     if args.api == 'local':
         app = create_app(args.model, args.attn_implementation)
         uvicorn.run(app, host='0.0.0.0', port=args.port, workers=1)

evalscope/perf/utils/log_utils.py

@@ -34,8 +34,15 @@ def init_swanlab(args: Arguments) -> None:
     current_time = datetime.datetime.now().strftime('%Y%m%d_%H%M%S')
     name = args.name if args.name else f'{args.model_id}_{current_time}'
     swanlab.config.update({'framework': '📏evalscope'})
-    swanlab.init(
-        project=os.getenv('SWANLAB_PROJ_NAME', 'perf_benchmark'),
-        name=name,
-        config=args.to_dict(),
-        mode='local' if args.swanlab_api_key == 'local' else None)
+    init_kwargs = {
+        'project': os.getenv('SWANLAB_PROJ_NAME', 'perf_benchmark'),
+        'name': name,
+        'config': args.to_dict(),
+        'mode': 'local' if args.swanlab_api_key == 'local' else None
+    }
+
+    workspace = os.getenv('SWANLAB_WORKSPACE')
+    if workspace:
+        init_kwargs['workspace'] = workspace
+
+    swanlab.init(**init_kwargs)

evalscope/perf/utils/rich_display.py

@@ -92,7 +92,7 @@ def print_summary(all_results, model_name):
     basic_info.add_row('Model', model_name)
     basic_info.add_row('Total Generated', f'{total_tokens:,} tokens')
     basic_info.add_row('Total Test Time', f'{total_time:.2f} seconds')
-    basic_info.add_row('Avg Output Rate', f'{total_tokens/total_time:.2f} tokens/sec')
+    basic_info.add_row('Avg Output Rate', f'{total_tokens / total_time:.2f} tokens/sec')
 
     console.print('\nBasic Information:')
     console.print(basic_info)

evalscope/report/__init__.py

@@ -1,6 +1,38 @@
 # Copyright (c) Alibaba, Inc. and its affiliates.
+from typing import TYPE_CHECKING
 
-from evalscope.report.app_arguments import add_argument
-from evalscope.report.combinator import gen_table, get_data_frame, get_report_list
-from evalscope.report.generator import ReportGenerator
-from evalscope.report.utils import Category, Report, ReportKey, Subset
+from evalscope.utils.import_utils import _LazyModule
+
+if TYPE_CHECKING:
+    from .combinator import gen_report_table, gen_table, get_data_frame, get_report_list
+    from .generator import ReportGenerator
+    from .utils import Category, Report, ReportKey, Subset
+
+else:
+    _import_structure = {
+        'combinator': [
+            'gen_table',
+            'get_data_frame',
+            'get_report_list',
+            'gen_report_table',
+        ],
+        'generator': [
+            'ReportGenerator',
+        ],
+        'utils': [
+            'Category',
+            'Report',
+            'ReportKey',
+            'Subset',
+        ],
+    }
+
+    import sys
+
+    sys.modules[__name__] = _LazyModule(
+        __name__,
+        globals()['__file__'],
+        _import_structure,
+        module_spec=__spec__,
+        extra_objects={},
+    )

evalscope/report/combinator.py

@@ -48,6 +48,14 @@ def gen_table(reports_path_list: list) -> str:
     return tabulate(table, headers=table.columns, tablefmt='grid', showindex=False)
 
 
+def gen_report_table(report: Report) -> str:
+    """
+    Generate a report table for a single report.
+    """
+    table = report.to_dataframe(flatten_metrics=True, flatten_categories=True)
+    return tabulate(table, headers=table.columns, tablefmt='grid', showindex=False)
+
+
 class ReportsRecorder:
     COMMON_DATASET_PATH = []
     CUSTOM_DATASET_PATH = []

evalscope/report/generator.py

@@ -1,24 +1,42 @@
 import pandas as pd
 from pandas import DataFrame
+from typing import TYPE_CHECKING
 
 from evalscope.constants import DataCollection
 from evalscope.report.utils import *
 
+if TYPE_CHECKING:
+    from evalscope.benchmarks import DataAdapter
+
 
 class ReportGenerator:
 
     @staticmethod
-    def gen_report(subset_score_map: dict, report_name: str, **kwargs) -> Report:
+    def gen_report(subset_score_map: dict, model_name: str, data_adapter: 'DataAdapter', **kwargs) -> Report:
         """
-        Generate report for specific dataset.
-        subset_score_map: e.g. {subset_name: [{'metric_name': 'AverageAccuracy', 'score': 0.3389, 'num': 100}, {'metric_name': 'WeightedAverageAccuracy', 'score': 0.3389, 'num': 100}]}
-        category_map: e.g. {'subset_name': ['category_name1', 'category_name2'], ...}
-        metric_list: e.g. [{'object': AverageAccuracy, 'name': 'AverageAccuracy'}, {'object': 'WeightedAverageAccuracy', 'name': 'WeightedAverageAccuracy'}]
+        Generate a report for a specific dataset based on provided subset scores.
+
+        Args:
+            subset_score_map (dict): A mapping from subset names to a list of score dictionaries.
+                    {
+                        'subset_name': [
+                            {'metric_name': 'AverageAccuracy', 'score': 0.3389, 'num': 100},
+                            {'metric_name': 'WeightedAverageAccuracy', 'score': 0.3389, 'num': 100}
+                        ],
+                        ...
+                    }
+            report_name (str): The name of the report to generate.
+            data_adapter (DataAdapter): An adapter object for data handling.
+
+        Returns:
+            Report: A structured report object containing metrics, categories, and subsets.
+
+            >>> report = gen_report(subset_score_map, "My Report", data_adapter, dataset_name="Dataset", model_name="Model")
         """  # noqa: E501
 
-        dataset_name = kwargs.get('dataset_name', None)
-        model_name = kwargs.get('model_name', None)
-        category_map = kwargs.get('category_map', {})
+        dataset_name = data_adapter.name
+        category_map = data_adapter.category_map
+        report_name = f'{model_name}@{dataset_name}'
 
         def flatten_subset() -> DataFrame:
             """
@@ -59,7 +77,13 @@ class ReportGenerator:
 
             metrics_list.append(Metric(name=metric_name, categories=categories))
 
-        report = Report(name=report_name, metrics=metrics_list, dataset_name=dataset_name, model_name=model_name)
+        report = Report(
+            name=report_name,
+            metrics=metrics_list,
+            dataset_name=dataset_name,
+            model_name=model_name,
+            dataset_description=data_adapter.description,
+            dataset_pretty_name=data_adapter.pretty_name)
         return report
 
     @staticmethod

evalscope/report/utils.py

@@ -1,4 +1,5 @@
 import json
+import os
 import pandas as pd
 from collections import defaultdict
 from dataclasses import asdict, dataclass, field
@@ -6,6 +7,9 @@ from typing import Any, Dict, List
 
 from evalscope.metrics import macro_mean, micro_mean
 from evalscope.utils import normalize_score
+from evalscope.utils.logger import get_logger
+
+logger = get_logger()
 
 
 @dataclass
@@ -70,13 +74,28 @@ class ReportKey:
     score = 'Score'
 
 
+ANALYSIS_PROMPT = """根据给出的json格式的模型评测结果，输出分析报告，要求如下：
+1. 报告分为 总体表现、关键指标分析、改进建议、结论 四部分
+2. 若模型有多种指标，将其分为低分、中分、高分三个部分，并列出markdown表格
+3. 只列出报告本身，不要有其他多余内容
+4. 输出报告语言为{language}
+
+```json
+{report_str}
+```
+"""
+
+
 @dataclass
 class Report:
     name: str = 'default_report'
     dataset_name: str = 'default_dataset'
+    dataset_pretty_name: str = ''
+    dataset_description: str = ''
     model_name: str = 'default_model'
     score: float = 0.0
     metrics: List[Metric] = field(default_factory=list)
+    analysis: str = 'N/A'
 
     def __post_init__(self):
         self.score = self.metrics[0].score  # NOTE: only use the first metric by default
@@ -84,15 +103,29 @@ class Report:
     def to_dict(self) -> Dict[str, Any]:
         return asdict(self)
 
+    def to_json_str(self) -> str:
+        return json.dumps(self.to_dict(), indent=4, ensure_ascii=False)
+
+    def to_json(self, json_file: str):
+        # ensure the directory exists
+        os.makedirs(os.path.dirname(json_file), exist_ok=True)
+        # write the report to a json file
+        with open(json_file, 'w', encoding='utf-8') as f:
+            json.dump(self.to_dict(), f, indent=4, ensure_ascii=False)
+
     @classmethod
     def from_dict(cls, data: dict):
         metrics = [Metric.from_dict(metric) for metric in data.get('metrics', [])]
         return cls(
             name=data['name'],
+            dataset_name=data['dataset_name'],
+            dataset_pretty_name=data.get('dataset_pretty_name'),
+            dataset_description=data.get('dataset_description'),
             score=data['score'],
+            model_name=data['model_name'],
             metrics=metrics,
-            dataset_name=data['dataset_name'],
-            model_name=data['model_name'])
+            analysis=data.get('analysis', 'N/A'),
+        )
 
     @classmethod
     def from_json(cls, json_file: str):
@@ -111,7 +144,7 @@ class Report:
                     table[ReportKey.category_name].append(category.name)
                     table[ReportKey.subset_name].append(subset.name)
                     table[ReportKey.num].append(subset.num)
-                    table[ReportKey.score].append(subset.score)  # TODO: convert to percentage
+                    table[ReportKey.score].append(subset.score)
             # NOTE: only flatten metrics if needed, use the first metric by default
             if not flatten_metrics:
                 break
@@ -131,3 +164,27 @@ class Report:
 
         df_categories.drop(columns=[ReportKey.category_name], inplace=True)
         return df_categories
+
+    def generate_analysis(self, judge_llm_config: dict) -> str:
+        import locale
+
+        from evalscope.metrics import LLMJudge
+
+        try:
+            # get the default locale
+            lang, _ = locale.getlocale()
+
+            if lang is None:
+                language = '中文'
+            else:
+                language = 'en' if lang.startswith('en') else '中文'
+
+            prompt = ANALYSIS_PROMPT.format(language=language, report_str=self.to_json_str())
+            judge_llm = LLMJudge(**judge_llm_config)
+            response = judge_llm(prompt)
+        except Exception as e:
+            logger.error(f'Error generating analysis: {e}')
+            response = 'N/A'
+
+        self.analysis = response
+        return response

evalscope/run.py

@@ -43,6 +43,9 @@ def run_single_task(task_cfg: TaskConfig, run_time: str) -> dict:
     else:
         result = evaluate_model(task_cfg, outputs)
 
+        logger.info(f'Finished evaluation for {task_cfg.model_id} on {task_cfg.datasets}')
+        logger.info(f'Output directory: {outputs.outputs_dir}')
+
     return result
 
 
@@ -109,6 +112,7 @@ def get_backend_manager_class(eval_backend: EvalBackend):
 def evaluate_model(task_cfg: TaskConfig, outputs: OutputsStructure) -> dict:
     """Evaluate the model based on the provided task configuration."""
     from evalscope.models import get_local_model
+    from evalscope.report import gen_table
 
     # Initialize evaluator
     eval_results = {}
@@ -122,10 +126,18 @@ def evaluate_model(task_cfg: TaskConfig, outputs: OutputsStructure) -> dict:
     task_cfg.dump_yaml(outputs.configs_dir)
     logger.info(task_cfg)
 
+    # Run evaluation for each evaluator
     for evaluator in evaluators:
         res_dict = evaluator.eval()
         eval_results[evaluator.dataset_name] = res_dict
 
+    # Make overall report
+    try:
+        report_table: str = gen_table([outputs.reports_dir])
+        logger.info(f'Overall report table: \n{report_table} \n')
+    except Exception:
+        logger.error('Failed to generate report table.')
+
     # Clean up
     if base_model is not None:
         import gc