evalscope 0.16.2__py3-none-any.whl → 0.17.0__py3-none-any.whl

evalscope/app/ui/visualization.py

@@ -0,0 +1,36 @@
+"""
+Visualization components for the Evalscope dashboard.
+"""
+import gradio as gr
+from dataclasses import dataclass
+from typing import TYPE_CHECKING
+
+from ..utils.localization import get_visualization_locale
+from .multi_model import MultiModelComponents, create_multi_model_tab
+from .single_model import SingleModelComponents, create_single_model_tab
+
+if TYPE_CHECKING:
+    from .sidebar import SidebarComponents
+
+
+@dataclass
+class VisualizationComponents:
+    single_model: SingleModelComponents
+    multi_model: MultiModelComponents
+
+
+def create_visualization(sidebar: 'SidebarComponents', lang: str):
+    locale_dict = get_visualization_locale(lang)
+
+    with gr.Column(visible=True):
+        gr.Markdown(f'## {locale_dict["visualization"]}')
+        with gr.Tabs():
+            with gr.Tab(locale_dict['single_model']):
+                single = create_single_model_tab(sidebar, lang)
+
+            with gr.Tab(locale_dict['multi_model']):
+                multi = create_multi_model_tab(sidebar, lang)
+    return VisualizationComponents(
+        single_model=single,
+        multi_model=multi,
+    )

evalscope/app/utils/data_utils.py

@@ -0,0 +1,178 @@
+"""
+Data loading and processing utilities for the Evalscope dashboard.
+"""
+import glob
+import numpy as np
+import os
+import pandas as pd
+from typing import Any, Dict, List, Union
+
+from evalscope.constants import DataCollection
+from evalscope.report import Report, ReportKey, get_data_frame, get_report_list
+from evalscope.utils.io_utils import OutputsStructure, yaml_to_dict
+from evalscope.utils.logger import get_logger
+from ..constants import DATASET_TOKEN, MODEL_TOKEN, REPORT_TOKEN
+
+logger = get_logger()
+
+
+def scan_for_report_folders(root_path):
+    """Scan for folders containing reports subdirectories"""
+    logger.debug(f'Scanning for report folders in {root_path}')
+    if not os.path.exists(root_path):
+        return []
+
+    reports = []
+    # Iterate over all folders in the root path
+    for folder in glob.glob(os.path.join(root_path, '*')):
+        # Check if reports folder exists
+        reports_path = os.path.join(folder, OutputsStructure.REPORTS_DIR)
+        if not os.path.exists(reports_path):
+            continue
+
+        # Iterate over all items in reports folder
+        for model_item in glob.glob(os.path.join(reports_path, '*')):
+            if not os.path.isdir(model_item):
+                continue
+            datasets = []
+            for dataset_item in glob.glob(os.path.join(model_item, '*.json')):
+                datasets.append(os.path.splitext(os.path.basename(dataset_item))[0])
+            datasets = DATASET_TOKEN.join(datasets)
+            reports.append(
+                f'{os.path.basename(folder)}{REPORT_TOKEN}{os.path.basename(model_item)}{MODEL_TOKEN}{datasets}')
+
+    reports = sorted(reports, reverse=True)
+    logger.debug(f'reports: {reports}')
+    return reports
+
+
+def process_report_name(report_name: str):
+    prefix, report_name = report_name.split(REPORT_TOKEN)
+    model_name, datasets = report_name.split(MODEL_TOKEN)
+    datasets = datasets.split(DATASET_TOKEN)
+    return prefix, model_name, datasets
+
+
+def load_single_report(root_path: str, report_name: str):
+    prefix, model_name, datasets = process_report_name(report_name)
+    report_path_list = os.path.join(root_path, prefix, OutputsStructure.REPORTS_DIR, model_name)
+    report_list = get_report_list([report_path_list])
+
+    config_files = glob.glob(os.path.join(root_path, prefix, OutputsStructure.CONFIGS_DIR, '*.yaml'))
+    if not config_files:
+        raise FileNotFoundError(
+            f'No configuration files found in {os.path.join(root_path, prefix, OutputsStructure.CONFIGS_DIR)}')
+    task_cfg_path = config_files[0]
+    task_cfg = yaml_to_dict(task_cfg_path)
+    return report_list, datasets, task_cfg
+
+
+def load_multi_report(root_path: str, report_names: List[str]):
+    report_list = []
+    for report_name in report_names:
+        prefix, model_name, datasets = process_report_name(report_name)
+        report_path_list = os.path.join(root_path, prefix, OutputsStructure.REPORTS_DIR, model_name)
+        reports = get_report_list([report_path_list])
+        report_list.extend(reports)
+    return report_list
+
+
+def get_acc_report_df(report_list: List[Report]):
+    data_dict = []
+    for report in report_list:
+        if report.name == DataCollection.NAME:
+            for metric in report.metrics:
+                for category in metric.categories:
+                    item = {
+                        ReportKey.model_name: report.model_name,
+                        ReportKey.dataset_name: '/'.join(category.name),
+                        ReportKey.score: category.score,
+                        ReportKey.num: category.num,
+                    }
+                    data_dict.append(item)
+        else:
+            item = {
+                ReportKey.model_name: report.model_name,
+                ReportKey.dataset_name: report.dataset_name,
+                ReportKey.score: report.score,
+                ReportKey.num: report.metrics[0].num,
+            }
+            data_dict.append(item)
+    df = pd.DataFrame.from_dict(data_dict, orient='columns')
+
+    styler = style_df(df, columns=[ReportKey.score])
+    return df, styler
+
+
+def style_df(df: pd.DataFrame, columns: List[str] = None):
+    # Apply background gradient to the specified columns
+    styler = df.style.background_gradient(subset=columns, cmap='RdYlGn', vmin=0.0, vmax=1.0, axis=0)
+    # Format the dataframe with a precision of 4 decimal places
+    styler.format(precision=4)
+    return styler
+
+
+def get_compare_report_df(acc_df: pd.DataFrame):
+    df = acc_df.pivot_table(index=ReportKey.model_name, columns=ReportKey.dataset_name, values=ReportKey.score)
+    df.reset_index(inplace=True)
+
+    styler = style_df(df)
+    return df, styler
+
+
+def get_single_dataset_df(df: pd.DataFrame, dataset_name: str):
+    df = df[df[ReportKey.dataset_name] == dataset_name]
+    styler = style_df(df, columns=[ReportKey.score])
+    return df, styler
+
+
+def get_report_analysis(report_list: List[Report], dataset_name: str) -> str:
+    for report in report_list:
+        if report.dataset_name == dataset_name:
+            return report.analysis
+    return 'N/A'
+
+
+def get_model_prediction(work_dir: str, model_name: str, dataset_name: str, subset_name: str):
+    data_path = os.path.join(work_dir, OutputsStructure.REVIEWS_DIR, model_name)
+    subset_name = subset_name.replace('/', '_')  # for collection report
+    review_path = os.path.join(data_path, f'{dataset_name}_{subset_name}.jsonl')
+    logger.debug(f'review_path: {review_path}')
+    origin_df = pd.read_json(review_path, lines=True)
+
+    ds = []
+    for i, item in origin_df.iterrows():
+        raw_input = item['raw_input']
+        sample_index = item['index']
+        for choice_index, choice in enumerate(item['choices']):
+            raw_pred_answer = choice['message']['content']
+            parsed_gold_answer = choice['review']['gold']
+            parsed_pred_answer = choice['review']['pred']
+            score = choice['review']['result']
+            raw_d = {
+                'Index': f'{sample_index}_{choice_index}',
+                'Input': raw_input,
+                'Generated': raw_pred_answer if raw_pred_answer != parsed_pred_answer else '*Same as Pred*',
+                'Gold': parsed_gold_answer if parsed_gold_answer != raw_input else '*Same as Input*',
+                'Pred': parsed_pred_answer,
+                'Score': score,
+                'NScore': normalize_score(score)
+            }
+            ds.append(raw_d)
+
+    df_subset = pd.DataFrame(ds)
+    return df_subset
+
+
+def normalize_score(score):
+    try:
+        if isinstance(score, bool):
+            return 1.0 if score else 0.0
+        elif isinstance(score, dict):
+            for key in score:
+                return float(score[key])
+            return 0.0
+        else:
+            return float(score)
+    except (ValueError, TypeError):
+        return 0.0

evalscope/app/utils/localization.py

@@ -0,0 +1,221 @@
+"""
+Localization utilities for the Evalscope dashboard.
+"""
+from typing import Any, Dict
+
+
+def get_sidebar_locale(lang: str) -> Dict[str, str]:
+    locale_dict = {
+        'settings': {
+            'zh': '设置',
+            'en': 'Settings'
+        },
+        'report_root_path': {
+            'zh': '报告根路径',
+            'en': 'Report Root Path'
+        },
+        'select_reports': {
+            'zh': '请选择报告',
+            'en': 'Select Reports'
+        },
+        'load_btn': {
+            'zh': '加载并查看',
+            'en': 'Load & View'
+        },
+        'note': {
+            'zh': '请选择报告并点击`加载并查看`来查看数据',
+            'en': 'Please select reports and click `Load & View` to view the data'
+        },
+        'warning': {
+            'zh': '没有找到报告，请检查路径',
+            'en': 'No reports found, please check the path'
+        }
+    }
+    return {k: v[lang] for k, v in locale_dict.items()}
+
+
+def get_visualization_locale(lang: str) -> Dict[str, str]:
+    locale_dict = {
+        'visualization': {
+            'zh': '可视化',
+            'en': 'Visualization'
+        },
+        'single_model': {
+            'zh': '单模型',
+            'en': 'Single Model'
+        },
+        'multi_model': {
+            'zh': '多模型',
+            'en': 'Multi Model'
+        }
+    }
+    return {k: v[lang] for k, v in locale_dict.items()}
+
+
+def get_single_model_locale(lang: str) -> Dict[str, str]:
+    locale_dict = {
+        'select_report': {
+            'zh': '选择报告',
+            'en': 'Select Report'
+        },
+        'task_config': {
+            'zh': '任务配置',
+            'en': 'Task Config'
+        },
+        'datasets_overview': {
+            'zh': '数据集概览',
+            'en': 'Datasets Overview'
+        },
+        'dataset_components': {
+            'zh': '数据集组成',
+            'en': 'Dataset Components'
+        },
+        'dataset_scores': {
+            'zh': '数据集分数',
+            'en': 'Dataset Scores'
+        },
+        'report_analysis': {
+            'zh': '报告智能分析',
+            'en': 'Report Intelligent Analysis'
+        },
+        'dataset_scores_table': {
+            'zh': '数据集分数表',
+            'en': 'Dataset Scores Table'
+        },
+        'dataset_details': {
+            'zh': '数据集详情',
+            'en': 'Dataset Details'
+        },
+        'select_dataset': {
+            'zh': '选择数据集',
+            'en': 'Select Dataset'
+        },
+        'model_prediction': {
+            'zh': '模型预测',
+            'en': 'Model Prediction'
+        },
+        'select_subset': {
+            'zh': '选择子集',
+            'en': 'Select Subset'
+        },
+        'answer_mode': {
+            'zh': '答案模式',
+            'en': 'Answer Mode'
+        },
+        'page': {
+            'zh': '页码',
+            'en': 'Page'
+        },
+        'score_threshold': {
+            'zh': '分数阈值',
+            'en': 'Score Threshold'
+        },
+    }
+    return {k: v[lang] for k, v in locale_dict.items()}
+
+
+def get_multi_model_locale(lang: str) -> Dict[str, str]:
+    locale_dict = {
+        'select_reports': {
+            'zh': '请选择报告',
+            'en': 'Select Reports'
+        },
+        'models_overview': {
+            'zh': '模型概览',
+            'en': 'Models Overview'
+        },
+        'model_radar': {
+            'zh': '模型对比雷达',
+            'en': 'Model Comparison Radar'
+        },
+        'model_scores': {
+            'zh': '模型对比分数',
+            'en': 'Model Comparison Scores'
+        },
+        'model_comparison_details': {
+            'zh': '模型对比详情',
+            'en': 'Model Comparison Details'
+        },
+        'select_model_a': {
+            'zh': '选择模型A',
+            'en': 'Select Model A'
+        },
+        'select_model_b': {
+            'zh': '选择模型B',
+            'en': 'Select Model B'
+        },
+        'select_dataset': {
+            'zh': '选择数据集',
+            'en': 'Select Dataset'
+        },
+        'model_predictions': {
+            'zh': '模型预测',
+            'en': 'Model Predictions'
+        },
+        'select_subset': {
+            'zh': '选择子集',
+            'en': 'Select Subset'
+        },
+        'answer_mode': {
+            'zh': '答案模式',
+            'en': 'Answer Mode'
+        },
+        'score_threshold': {
+            'zh': '分数阈值',
+            'en': 'Score Threshold'
+        },
+        'comparison_counts': {
+            'zh': '对比统计',
+            'en': 'Comparison Counts'
+        },
+        'page': {
+            'zh': '页码',
+            'en': 'Page'
+        },
+        'input': {
+            'zh': '输入',
+            'en': 'Input'
+        },
+        'gold_answer': {
+            'zh': '标准答案',
+            'en': 'Gold Answer'
+        },
+        'score': {
+            'zh': '分数',
+            'en': 'Score'
+        },
+        'normalized_score': {
+            'zh': '归一化分数',
+            'en': 'Normalized Score'
+        },
+        'prediction': {
+            'zh': '预测',
+            'en': 'Prediction'
+        },
+        'generated': {
+            'zh': '生成结果',
+            'en': 'Generated'
+        }
+    }
+    return {k: v[lang] for k, v in locale_dict.items()}
+
+
+def get_app_locale(lang: str) -> Dict[str, str]:
+    locale_dict = {
+        'title': {
+            'zh': '📈 EvalScope 看板',
+            'en': '📈 Evalscope Dashboard'
+        },
+        'star_beggar': {
+            'zh':
+            '喜欢<a href=\"https://github.com/modelscope/evalscope\" target=\"_blank\">EvalScope</a>就动动手指给我们加个star吧 🥺 ',
+            'en':
+            'If you like <a href=\"https://github.com/modelscope/evalscope\" target=\"_blank\">EvalScope</a>, '
+            'please take a few seconds to star us 🥺 '
+        },
+        'note': {
+            'zh': '请选择报告',
+            'en': 'Please select reports'
+        }
+    }
+    return {k: v[lang] for k, v in locale_dict.items()}

evalscope/app/utils/text_utils.py

@@ -0,0 +1,119 @@
+"""
+Text processing utilities for the Evalscope dashboard.
+"""
+import json
+import numpy as np
+import os
+import pandas as pd
+import re
+from typing import Any, Dict, List
+
+from evalscope.utils.logger import get_logger
+from ..constants import LATEX_DELIMITERS
+
+logger = get_logger()
+
+
+def convert_markdown_image(text):
+    if not os.path.isfile(text):
+        return text
+    # Convert the image path to a markdown image tag
+    if text.endswith('.png') or text.endswith('.jpg') or text.endswith('.jpeg'):
+        text = os.path.abspath(text)
+        image_tag = f'![image](gradio_api/file={text})'
+        logger.debug(f'Converting image path to markdown: {text} -> {image_tag}')
+        return image_tag
+    return text
+
+
+def convert_html_tags(text):
+    # match begin label
+    text = re.sub(r'<(\w+)>', r'[\1]', text)
+    # match end label
+    text = re.sub(r'</(\w+)>', r'[/\1]', text)
+    return text
+
+
+def process_string(string: str, max_length: int = 2048) -> str:
+    string = convert_html_tags(string)  # for display labels e.g.
+    if max_length and len(string) > max_length:
+        return f'{string[:max_length // 2]}......{string[-max_length // 2:]}'
+    return string
+
+
+def dict_to_markdown(data) -> str:
+    markdown_lines = []
+
+    for key, value in data.items():
+        bold_key = f'**{key}**'
+
+        if isinstance(value, list):
+            value_str = '\n' + '\n'.join([f'- {process_model_prediction(item, max_length=None)}' for item in value])
+        elif isinstance(value, dict):
+            value_str = dict_to_markdown(value)
+        else:
+            value_str = str(value)
+
+        value_str = process_string(value_str, max_length=None)  # Convert HTML tags but don't truncate
+        markdown_line = f'{bold_key}:\n{value_str}'
+        markdown_lines.append(markdown_line)
+
+    return '\n\n'.join(markdown_lines)
+
+
+def process_model_prediction_old(item: Any, max_length: int = 2048) -> str:
+    """
+    Process model prediction output into a formatted string.
+
+    Args:
+        item: The item to process. Can be a string, list, or dictionary.
+        max_length: The maximum length of the output string.
+
+    Returns:
+        A formatted string representation of the input.
+    """
+    if isinstance(item, dict):
+        result = dict_to_markdown(item)
+    elif isinstance(item, list):
+        result = '\n'.join([f'- {process_model_prediction(i, max_length=None)}' for i in item])
+    else:
+        result = str(item)
+
+    # Apply HTML tag conversion and truncation only at the final output
+    if max_length is not None:
+        return process_string(result, max_length)
+    return result
+
+
+def process_model_prediction(item: Any, max_length: int = 32000) -> str:
+    if isinstance(item, (dict, list)):
+        result = json.dumps(item, ensure_ascii=False, indent=2)
+        result = f'```json\n{result}\n```'
+    else:
+        result = str(item)
+
+    # Apply HTML tag conversion and truncation only at the final output
+    if max_length is not None:
+        return process_string(result, max_length)
+
+    return result
+
+
+def process_json_content(content: Any) -> str:
+    """
+    Process JSON content to convert it into a markdown-friendly format.
+
+    Args:
+        content (str): The JSON content as a string.
+
+    Returns:
+        str: The processed content formatted for markdown display.
+    """
+    if isinstance(content, (np.bool_, np.int_, np.float_)):
+        content = str(content)
+
+    if isinstance(content, str):
+        content = {'content': content}
+
+    content_json = json.dumps(content, ensure_ascii=False, indent=2)
+    return content_json

evalscope/app/utils/visualization.py

@@ -0,0 +1,91 @@
+"""
+Visualization utilities for the Evalscope dashboard.
+"""
+import numpy as np
+import pandas as pd
+import plotly.express as px
+import plotly.graph_objects as go
+from typing import List
+
+from evalscope.constants import DataCollection
+from evalscope.report import Report, ReportKey, get_data_frame
+from evalscope.utils.logger import get_logger
+from ..constants import DEFAULT_BAR_WIDTH, PLOTLY_THEME
+
+logger = get_logger()
+
+
+def plot_single_report_scores(df: pd.DataFrame):
+    if df is None:
+        return None
+    logger.debug(f'df: {df}')
+    plot = px.bar(df, x=df[ReportKey.dataset_name], y=df[ReportKey.score], text=df[ReportKey.score])
+
+    width = DEFAULT_BAR_WIDTH if len(df[ReportKey.dataset_name]) <= 5 else None
+    plot.update_traces(width=width, texttemplate='%{text:.2f}', textposition='outside')
+    plot.update_layout(uniformtext_minsize=12, uniformtext_mode='hide', yaxis=dict(range=[0, 1]), template=PLOTLY_THEME)
+    return plot
+
+
+def plot_single_report_sunburst(report_list: List[Report]):
+    if report_list[0].name == DataCollection.NAME:
+        df = get_data_frame(report_list=report_list)
+        categories = sorted([i for i in df.columns if i.startswith(ReportKey.category_prefix)])
+        path = categories + [ReportKey.subset_name]
+    else:
+        df = get_data_frame(report_list=report_list, flatten_metrics=False)
+        categories = sorted([i for i in df.columns if i.startswith(ReportKey.category_prefix)])
+        path = [ReportKey.dataset_name] + categories + [ReportKey.subset_name]
+    logger.debug(f'df: {df}')
+    df[categories] = df[categories].fillna('default')  # NOTE: fillna for empty categories
+
+    plot = px.sunburst(
+        df,
+        path=path,
+        values=ReportKey.num,
+        color=ReportKey.score,
+        color_continuous_scale='RdYlGn',  # see https://plotly.com/python/builtin-colorscales/
+        color_continuous_midpoint=np.average(df[ReportKey.score], weights=df[ReportKey.num]),
+        template=PLOTLY_THEME,
+        maxdepth=4)
+    plot.update_traces(insidetextorientation='radial')
+    plot.update_layout(margin=dict(t=10, l=10, r=10, b=10), coloraxis=dict(cmin=0, cmax=1), height=600)
+    return plot
+
+
+def plot_single_dataset_scores(df: pd.DataFrame):
+    # TODO: add metric radio and replace category name
+    plot = px.bar(
+        df,
+        x=df[ReportKey.metric_name],
+        y=df[ReportKey.score],
+        color=df[ReportKey.subset_name],
+        text=df[ReportKey.score],
+        barmode='group')
+
+    width = 0.2 if len(df[ReportKey.subset_name]) <= 3 else None
+    plot.update_traces(width=width, texttemplate='%{text:.2f}', textposition='outside')
+    plot.update_layout(uniformtext_minsize=12, uniformtext_mode='hide', yaxis=dict(range=[0, 1]), template=PLOTLY_THEME)
+    return plot
+
+
+def plot_multi_report_radar(df: pd.DataFrame):
+    fig = go.Figure()
+
+    grouped = df.groupby(ReportKey.model_name)
+    common_datasets = set.intersection(*[set(group[ReportKey.dataset_name]) for _, group in grouped])
+
+    for model_name, group in grouped:
+        common_group = group[group[ReportKey.dataset_name].isin(common_datasets)]
+        fig.add_trace(
+            go.Scatterpolar(
+                r=common_group[ReportKey.score],
+                theta=common_group[ReportKey.dataset_name],
+                name=model_name,
+                fill='toself'))
+
+    fig.update_layout(
+        template=PLOTLY_THEME,
+        polar=dict(radialaxis=dict(visible=True, range=[0, 1])),
+        margin=dict(t=20, l=20, r=20, b=20))
+    return fig

evalscope/backend/opencompass/backend_manager.py

@@ -8,7 +8,8 @@ from typing import Optional, Union
 
 from evalscope.backend.base import BackendManager
 from evalscope.backend.opencompass.api_meta_template import get_template
-from evalscope.utils import get_module_path, get_valid_list, is_module_installed
+from evalscope.utils.import_utils import get_module_path, is_module_installed
+from evalscope.utils.io_utils import get_valid_list
 from evalscope.utils.logger import get_logger
 
 logger = get_logger()

evalscope/backend/rag_eval/backend_manager.py

@@ -2,7 +2,8 @@ import os
 from typing import Optional, Union
 
 from evalscope.backend.base import BackendManager
-from evalscope.utils import get_valid_list, is_module_installed
+from evalscope.utils.import_utils import is_module_installed
+from evalscope.utils.io_utils import get_valid_list
 from evalscope.utils.logger import get_logger
 
 logger = get_logger()

evalscope/backend/rag_eval/utils/embedding.py

@@ -12,8 +12,8 @@ from typing import Dict, List, Optional, Union
 
 from evalscope.backend.rag_eval.utils.tools import download_model
 from evalscope.constants import HubType
+from evalscope.utils.argument_utils import get_supported_params
 from evalscope.utils.logger import get_logger
-from evalscope.utils.utils import get_supported_params
 
 logger = get_logger()