evalscope 0.10.0__py3-none-any.whl → 1.2.0__py3-none-any.whl

evalscope/app/utils/data_utils.py

@@ -0,0 +1,195 @@
+"""
+Data loading and processing utilities for the Evalscope dashboard.
+"""
+import glob
+import os
+import pandas as pd
+from typing import Any, Dict, List, Union
+
+from evalscope.api.evaluator import CacheManager, ReviewResult
+from evalscope.constants import DataCollection
+from evalscope.report import Report, ReportKey, get_data_frame, get_report_list
+from evalscope.utils.io_utils import OutputsStructure, jsonl_to_list, yaml_to_dict
+from evalscope.utils.logger import get_logger
+from ..constants import DATASET_TOKEN, MODEL_TOKEN, REPORT_TOKEN
+
+logger = get_logger()
+
+
+def scan_for_report_folders(root_path):
+    """Scan for folders containing reports subdirectories"""
+    logger.debug(f'Scanning for report folders in {root_path}')
+    if not os.path.exists(root_path):
+        return []
+
+    reports = []
+    # Iterate over all folders in the root path
+    for folder in glob.glob(os.path.join(root_path, '*')):
+        # Check if reports folder exists
+        reports_path = os.path.join(folder, OutputsStructure.REPORTS_DIR)
+        if not os.path.exists(reports_path):
+            continue
+
+        # Iterate over all items in reports folder
+        for model_item in glob.glob(os.path.join(reports_path, '*')):
+            if not os.path.isdir(model_item):
+                continue
+            datasets = []
+            for dataset_item in glob.glob(os.path.join(model_item, '*.json')):
+                datasets.append(os.path.splitext(os.path.basename(dataset_item))[0])
+            datasets = DATASET_TOKEN.join(datasets)
+            reports.append(
+                f'{os.path.basename(folder)}{REPORT_TOKEN}{os.path.basename(model_item)}{MODEL_TOKEN}{datasets}'
+            )
+
+    reports = sorted(reports, reverse=True)
+    logger.debug(f'reports: {reports}')
+    return reports
+
+
+def process_report_name(report_name: str):
+    prefix, report_name = report_name.split(REPORT_TOKEN)
+    model_name, datasets = report_name.split(MODEL_TOKEN)
+    datasets = datasets.split(DATASET_TOKEN)
+    return prefix, model_name, datasets
+
+
+def load_single_report(root_path: str, report_name: str):
+    prefix, model_name, datasets = process_report_name(report_name)
+    report_path_list = os.path.join(root_path, prefix, OutputsStructure.REPORTS_DIR, model_name)
+    report_list = get_report_list([report_path_list])
+
+    config_files = glob.glob(os.path.join(root_path, prefix, OutputsStructure.CONFIGS_DIR, '*.yaml'))
+    if not config_files:
+        raise FileNotFoundError(
+            f'No configuration files found in {os.path.join(root_path, prefix, OutputsStructure.CONFIGS_DIR)}'
+        )
+    task_cfg_path = config_files[0]
+    task_cfg = yaml_to_dict(task_cfg_path)
+    return report_list, datasets, task_cfg
+
+
+def load_multi_report(root_path: str, report_names: List[str]):
+    report_list = []
+    for report_name in report_names:
+        prefix, model_name, datasets = process_report_name(report_name)
+        report_path_list = os.path.join(root_path, prefix, OutputsStructure.REPORTS_DIR, model_name)
+        reports = get_report_list([report_path_list])
+        report_list.extend(reports)
+    return report_list
+
+
+def get_acc_report_df(report_list: List[Report]):
+    data_dict = []
+    for report in report_list:
+        if report.name == DataCollection.NAME:
+            for metric in report.metrics:
+                for category in metric.categories:
+                    item = {
+                        ReportKey.model_name: report.model_name,
+                        ReportKey.dataset_name: '/'.join(category.name),
+                        ReportKey.score: category.score,
+                        ReportKey.num: category.num,
+                    }
+                    data_dict.append(item)
+        else:
+            item = {
+                ReportKey.model_name: report.model_name,
+                ReportKey.dataset_name: report.dataset_name,
+                ReportKey.score: report.score,
+                ReportKey.num: report.metrics[0].num,
+            }
+            data_dict.append(item)
+    df = pd.DataFrame.from_dict(data_dict, orient='columns')
+
+    styler = style_df(df, columns=[ReportKey.score])
+    return df, styler
+
+
+def style_df(df: pd.DataFrame, columns: List[str] = None):
+    # Apply background gradient to the specified columns
+    styler = df.style.background_gradient(subset=columns, cmap='RdYlGn', vmin=0.0, vmax=1.0, axis=0)
+    # Format the dataframe with a precision of 4 decimal places
+    styler.format(precision=4)
+    return styler
+
+
+def get_compare_report_df(acc_df: pd.DataFrame):
+    df = acc_df.pivot_table(index=ReportKey.model_name, columns=ReportKey.dataset_name, values=ReportKey.score)
+    df.reset_index(inplace=True)
+
+    styler = style_df(df)
+    return df, styler
+
+
+def get_single_dataset_df(df: pd.DataFrame, dataset_name: str):
+    df = df[df[ReportKey.dataset_name] == dataset_name]
+    styler = style_df(df, columns=[ReportKey.score])
+    return df, styler
+
+
+def get_report_analysis(report_list: List[Report], dataset_name: str) -> str:
+    for report in report_list:
+        if report.dataset_name == dataset_name:
+            return report.analysis
+    return 'N/A'
+
+
+def get_model_prediction(work_dir: str, model_name: str, dataset_name: str, subset_name: str):
+    # Load review cache
+    outputs = OutputsStructure(work_dir, is_make=False)
+    cache_manager = CacheManager(outputs, model_name, dataset_name)
+    if dataset_name == DataCollection.NAME:
+        review_cache_path = cache_manager.get_review_cache_path('default')
+    else:
+        review_cache_path = cache_manager.get_review_cache_path(subset_name)
+    logger.debug(f'review_path: {review_cache_path}')
+    review_caches = jsonl_to_list(review_cache_path)
+
+    ds = []
+    for cache in review_caches:
+        review_result = ReviewResult.model_validate(cache)
+        sample_score = review_result.sample_score
+
+        if dataset_name == DataCollection.NAME:
+            # Filter subset name
+            collection_info = sample_score.sample_metadata[DataCollection.INFO]
+            sample_dataset_name = collection_info.get('dataset_name', 'default')
+            sample_subset_name = collection_info.get('subset_name', 'default')
+            if f'{sample_dataset_name}/{sample_subset_name}' != subset_name:
+                continue
+
+        score = sample_score.score
+        metadata = sample_score.sample_metadata
+        prediction = score.prediction
+        target = review_result.target
+        extracted_prediction = score.extracted_prediction
+        raw_d = {
+            'Index': str(review_result.index),
+            'Input': review_result.input.replace('\n', '\n\n'),  # for markdown
+            'Metadata': metadata,
+            'Generated': prediction or '',  # Ensure no None value
+            'Gold': target,
+            'Pred': (extracted_prediction if extracted_prediction != prediction else '*Same as Generated*')
+            or '',  # Ensure no None value
+            'Score': score.model_dump(exclude_none=True),
+            'NScore': normalize_score(score.main_value)
+        }
+        ds.append(raw_d)
+
+    df_subset = pd.DataFrame(ds)
+    return df_subset
+
+
+def normalize_score(score):
+    try:
+        if isinstance(score, bool):
+            return 1.0 if score else 0.0
+        elif isinstance(score, dict):
+            for key in score:
+                return float(score[key])
+            return 0.0
+        else:
+            return float(score)
+    except (ValueError, TypeError):
+        return 0.0

evalscope/app/utils/env_utils.py

@@ -0,0 +1,12 @@
+# flake8: noqa
+import os
+
+
+def setup_env(args):
+    compat_dsw_gradio(args)
+
+
+def compat_dsw_gradio(args) -> None:
+    if ('JUPYTER_NAME' in os.environ) and ('dsw-'
+                                           in os.environ['JUPYTER_NAME']) and ('GRADIO_ROOT_PATH' not in os.environ):
+        os.environ['GRADIO_ROOT_PATH'] = f"/{os.environ['JUPYTER_NAME']}/proxy/{args.server_port}"

evalscope/app/utils/localization.py

@@ -0,0 +1,221 @@
+"""
+Localization utilities for the Evalscope dashboard.
+"""
+from typing import Any, Dict
+
+
+def get_sidebar_locale(lang: str) -> Dict[str, str]:
+    locale_dict = {
+        'settings': {
+            'zh': '设置',
+            'en': 'Settings'
+        },
+        'report_root_path': {
+            'zh': '报告根路径',
+            'en': 'Report Root Path'
+        },
+        'select_reports': {
+            'zh': '请选择报告',
+            'en': 'Select Reports'
+        },
+        'load_btn': {
+            'zh': '加载并查看',
+            'en': 'Load & View'
+        },
+        'note': {
+            'zh': '请选择报告并点击`加载并查看`来查看数据',
+            'en': 'Please select reports and click `Load & View` to view the data'
+        },
+        'warning': {
+            'zh': '没有找到报告，请检查路径',
+            'en': 'No reports found, please check the path'
+        }
+    }
+    return {k: v[lang] for k, v in locale_dict.items()}
+
+
+def get_visualization_locale(lang: str) -> Dict[str, str]:
+    locale_dict = {
+        'visualization': {
+            'zh': '可视化',
+            'en': 'Visualization'
+        },
+        'single_model': {
+            'zh': '单模型',
+            'en': 'Single Model'
+        },
+        'multi_model': {
+            'zh': '多模型',
+            'en': 'Multi Model'
+        }
+    }
+    return {k: v[lang] for k, v in locale_dict.items()}
+
+
+def get_single_model_locale(lang: str) -> Dict[str, str]:
+    locale_dict = {
+        'select_report': {
+            'zh': '选择报告',
+            'en': 'Select Report'
+        },
+        'task_config': {
+            'zh': '任务配置',
+            'en': 'Task Config'
+        },
+        'datasets_overview': {
+            'zh': '数据集概览',
+            'en': 'Datasets Overview'
+        },
+        'dataset_components': {
+            'zh': '数据集组成',
+            'en': 'Dataset Components'
+        },
+        'dataset_scores': {
+            'zh': '数据集分数',
+            'en': 'Dataset Scores'
+        },
+        'report_analysis': {
+            'zh': '报告智能分析',
+            'en': 'Report Intelligent Analysis'
+        },
+        'dataset_scores_table': {
+            'zh': '数据集分数表',
+            'en': 'Dataset Scores Table'
+        },
+        'dataset_details': {
+            'zh': '数据集详情',
+            'en': 'Dataset Details'
+        },
+        'select_dataset': {
+            'zh': '选择数据集',
+            'en': 'Select Dataset'
+        },
+        'model_prediction': {
+            'zh': '模型预测',
+            'en': 'Model Prediction'
+        },
+        'select_subset': {
+            'zh': '选择子集',
+            'en': 'Select Subset'
+        },
+        'answer_mode': {
+            'zh': '答案模式',
+            'en': 'Answer Mode'
+        },
+        'page': {
+            'zh': '页码',
+            'en': 'Page'
+        },
+        'score_threshold': {
+            'zh': '分数阈值',
+            'en': 'Score Threshold'
+        },
+    }
+    return {k: v[lang] for k, v in locale_dict.items()}
+
+
+def get_multi_model_locale(lang: str) -> Dict[str, str]:
+    locale_dict = {
+        'select_reports': {
+            'zh': '请选择报告',
+            'en': 'Select Reports'
+        },
+        'models_overview': {
+            'zh': '模型概览',
+            'en': 'Models Overview'
+        },
+        'model_radar': {
+            'zh': '模型对比雷达',
+            'en': 'Model Comparison Radar'
+        },
+        'model_scores': {
+            'zh': '模型对比分数',
+            'en': 'Model Comparison Scores'
+        },
+        'model_comparison_details': {
+            'zh': '模型对比详情',
+            'en': 'Model Comparison Details'
+        },
+        'select_model_a': {
+            'zh': '选择模型A',
+            'en': 'Select Model A'
+        },
+        'select_model_b': {
+            'zh': '选择模型B',
+            'en': 'Select Model B'
+        },
+        'select_dataset': {
+            'zh': '选择数据集',
+            'en': 'Select Dataset'
+        },
+        'model_predictions': {
+            'zh': '模型预测',
+            'en': 'Model Predictions'
+        },
+        'select_subset': {
+            'zh': '选择子集',
+            'en': 'Select Subset'
+        },
+        'answer_mode': {
+            'zh': '答案模式',
+            'en': 'Answer Mode'
+        },
+        'score_threshold': {
+            'zh': '分数阈值',
+            'en': 'Score Threshold'
+        },
+        'comparison_counts': {
+            'zh': '对比统计',
+            'en': 'Comparison Counts'
+        },
+        'page': {
+            'zh': '页码',
+            'en': 'Page'
+        },
+        'input': {
+            'zh': '输入',
+            'en': 'Input'
+        },
+        'gold_answer': {
+            'zh': '标准答案',
+            'en': 'Gold Answer'
+        },
+        'score': {
+            'zh': '分数',
+            'en': 'Score'
+        },
+        'normalized_score': {
+            'zh': '归一化分数',
+            'en': 'Normalized Score'
+        },
+        'prediction': {
+            'zh': '预测',
+            'en': 'Prediction'
+        },
+        'generated': {
+            'zh': '生成结果',
+            'en': 'Generated'
+        }
+    }
+    return {k: v[lang] for k, v in locale_dict.items()}
+
+
+def get_app_locale(lang: str) -> Dict[str, str]:
+    locale_dict = {
+        'title': {
+            'zh': '📈 EvalScope 看板',
+            'en': '📈 Evalscope Dashboard'
+        },
+        'star_beggar': {
+            'zh':
+            '喜欢<a href=\"https://github.com/modelscope/evalscope\" target=\"_blank\">EvalScope</a>就动动手指给我们加个star吧 🥺 ',
+            'en':
+            'If you like <a href=\"https://github.com/modelscope/evalscope\" target=\"_blank\">EvalScope</a>, '
+            'please take a few seconds to star us 🥺 '
+        },
+        'note': {
+            'zh': '请选择报告',
+            'en': 'Please select reports'
+        }
+    }
+    return {k: v[lang] for k, v in locale_dict.items()}

evalscope/app/utils/text_utils.py

@@ -0,0 +1,119 @@
+"""
+Text processing utilities for the Evalscope dashboard.
+"""
+import json
+import os
+import re
+from typing import Any, Dict, List, Optional
+
+from evalscope.utils.logger import get_logger
+from ..constants import LATEX_DELIMITERS
+
+logger = get_logger()
+
+
+def convert_markdown_image(text: str):
+    if text.startswith('data:image'):
+        # Convert base64 image data to a markdown image tag
+        image_tag = f'![image]({text})'
+        logger.debug(f'Converting base64 image data to markdown: {text[:30]}... -> {image_tag[:40]}...')
+        return image_tag
+    elif os.path.isfile(text):
+        # Convert the image path to a markdown image tag
+        if text.endswith('.png') or text.endswith('.jpg') or text.endswith('.jpeg'):
+            text = os.path.abspath(text)
+            image_tag = f'![image](gradio_api/file={text})'
+            logger.debug(f'Converting image path to markdown: {text} -> {image_tag}')
+            return image_tag
+    return text
+
+
+def convert_html_tags(text):
+    # match begin label
+    text = re.sub(r'<(\w+)>', r'[\1]', text)
+    # match end label
+    text = re.sub(r'</(\w+)>', r'[/\1]', text)
+    return text
+
+
+def process_string(string: str, max_length: int = 2048) -> str:
+    string = convert_html_tags(string)  # for display labels e.g.
+    if max_length and len(string) > max_length:
+        return f'{string[:max_length // 2]}......{string[-max_length // 2:]}'
+    return string
+
+
+def dict_to_markdown(data) -> str:
+    markdown_lines = []
+
+    for key, value in data.items():
+        bold_key = f'**{key}**'
+
+        if isinstance(value, list):
+            value_str = '\n' + '\n'.join([f'- {process_model_prediction(item, max_length=None)}' for item in value])
+        elif isinstance(value, dict):
+            value_str = dict_to_markdown(value)
+        else:
+            value_str = str(value)
+
+        value_str = process_string(value_str, max_length=None)  # Convert HTML tags but don't truncate
+        markdown_line = f'{bold_key}:\n{value_str}'
+        markdown_lines.append(markdown_line)
+
+    return '\n\n'.join(markdown_lines)
+
+
+def process_model_prediction_old(item: Any, max_length: int = 2048) -> str:
+    """
+    Process model prediction output into a formatted string.
+
+    Args:
+        item: The item to process. Can be a string, list, or dictionary.
+        max_length: The maximum length of the output string.
+
+    Returns:
+        A formatted string representation of the input.
+    """
+    if isinstance(item, dict):
+        result = dict_to_markdown(item)
+    elif isinstance(item, list):
+        result = '\n'.join([f'- {process_model_prediction(i, max_length=None)}' for i in item])
+    else:
+        result = str(item)
+
+    # Apply HTML tag conversion and truncation only at the final output
+    if max_length is not None:
+        return process_string(result, max_length)
+    return result
+
+
+def process_model_prediction(item: Any, max_length: Optional[int] = None) -> str:
+    if isinstance(item, (dict, list)):
+        result = json.dumps(item, ensure_ascii=False, indent=2)
+        result = f'```json\n{result}\n```'
+    else:
+        result = str(item)
+
+    # Apply HTML tag conversion and truncation only at the final output
+    if max_length is not None:
+        return process_string(result, max_length)
+
+    return result
+
+
+def process_json_content(content: Any) -> str:
+    """
+    Process JSON content to convert it into a markdown-friendly format.
+
+    Args:
+        content (str): The JSON content as a string.
+
+    Returns:
+        str: The processed content formatted for markdown display.
+    """
+
+    if isinstance(content, str):
+        content = {'content': content}
+
+    content_json = json.dumps(content, ensure_ascii=False, indent=2)
+    return content_json

evalscope/app/utils/visualization.py

@@ -0,0 +1,96 @@
+"""
+Visualization utilities for the Evalscope dashboard.
+"""
+import numpy as np
+import pandas as pd
+import plotly.express as px
+import plotly.graph_objects as go
+from typing import List
+
+from evalscope.constants import DataCollection
+from evalscope.report import Report, ReportKey, get_data_frame
+from evalscope.utils.logger import get_logger
+from ..constants import DEFAULT_BAR_WIDTH, PLOTLY_THEME
+
+logger = get_logger()
+
+
+def plot_single_report_scores(df: pd.DataFrame):
+    if df is None:
+        return None
+    logger.debug(f'df: \n{df}')
+    plot = px.bar(df, x=df[ReportKey.dataset_name], y=df[ReportKey.score], text=df[ReportKey.score])
+
+    width = DEFAULT_BAR_WIDTH if len(df[ReportKey.dataset_name]) <= 5 else None
+    plot.update_traces(width=width, texttemplate='%{text:.2f}', textposition='outside')
+    plot.update_layout(uniformtext_minsize=12, uniformtext_mode='hide', yaxis=dict(range=[0, 1]), template=PLOTLY_THEME)
+    return plot
+
+
+def plot_single_report_sunburst(report_list: List[Report]):
+    if report_list[0].name == DataCollection.NAME:
+        df = get_data_frame(report_list=report_list)
+        categories = sorted([i for i in df.columns if i.startswith(ReportKey.category_prefix)])
+        path = categories + [ReportKey.subset_name]
+    else:
+        df = get_data_frame(report_list=report_list, flatten_metrics=False)
+        categories = sorted([i for i in df.columns if i.startswith(ReportKey.category_prefix)])
+        path = [ReportKey.dataset_name] + categories + [ReportKey.subset_name]
+    logger.debug(f'df: \n{df}')
+    df[categories] = df[categories].fillna('default')  # NOTE: fillna for empty categories
+
+    plot = px.sunburst(
+        df,
+        path=path,
+        values=ReportKey.num,
+        color=ReportKey.score,
+        color_continuous_scale='RdYlGn',  # see https://plotly.com/python/builtin-colorscales/
+        color_continuous_midpoint=np.average(df[ReportKey.score], weights=df[ReportKey.num]),
+        template=PLOTLY_THEME,
+        maxdepth=4
+    )
+    plot.update_traces(insidetextorientation='radial')
+    plot.update_layout(margin=dict(t=10, l=10, r=10, b=10), coloraxis=dict(cmin=0, cmax=1), height=600)
+    return plot
+
+
+def plot_single_dataset_scores(df: pd.DataFrame):
+    # TODO: add metric radio and replace category name
+    plot = px.bar(
+        df,
+        x=df[ReportKey.metric_name],
+        y=df[ReportKey.score],
+        color=df[ReportKey.subset_name],
+        text=df[ReportKey.score],
+        barmode='group'
+    )
+
+    width = 0.2 if len(df[ReportKey.subset_name]) <= 3 else None
+    plot.update_traces(width=width, texttemplate='%{text:.2f}', textposition='outside')
+    plot.update_layout(uniformtext_minsize=12, uniformtext_mode='hide', yaxis=dict(range=[0, 1]), template=PLOTLY_THEME)
+    return plot
+
+
+def plot_multi_report_radar(df: pd.DataFrame):
+    fig = go.Figure()
+
+    grouped = df.groupby(ReportKey.model_name)
+    common_datasets = set.intersection(*[set(group[ReportKey.dataset_name]) for _, group in grouped])
+
+    for model_name, group in grouped:
+        common_group = group[group[ReportKey.dataset_name].isin(common_datasets)]
+        fig.add_trace(
+            go.Scatterpolar(
+                r=common_group[ReportKey.score],
+                theta=common_group[ReportKey.dataset_name],
+                name=model_name,
+                fill='toself'
+            )
+        )
+
+    fig.update_layout(
+        template=PLOTLY_THEME,
+        polar=dict(radialaxis=dict(visible=True, range=[0, 1])),
+        margin=dict(t=20, l=20, r=20, b=20)
+    )
+    return fig