evalscope 0.16.0__py3-none-any.whl → 0.16.1__py3-none-any.whl

evalscope/app/__init__.py

@@ -0,0 +1,28 @@
+# Copyright (c) Alibaba, Inc. and its affiliates.
+from typing import TYPE_CHECKING
+
+from evalscope.utils.import_utils import _LazyModule
+
+if TYPE_CHECKING:
+    from .app import create_app
+    from .arguments import add_argument
+
+else:
+    _import_structure = {
+        'app': [
+            'create_app',
+        ],
+        'arguments': [
+            'add_argument',
+        ],
+    }
+
+    import sys
+
+    sys.modules[__name__] = _LazyModule(
+        __name__,
+        globals()['__file__'],
+        _import_structure,
+        module_spec=__spec__,
+        extra_objects={},
+    )

evalscope/{report → app}/app.py

@@ -11,35 +11,15 @@ from dataclasses import dataclass
 from typing import Any, List, Union
 
 from evalscope.constants import DataCollection
-from evalscope.report import Report, ReportKey, add_argument, get_data_frame, get_report_list
+from evalscope.report import Report, ReportKey, get_data_frame, get_report_list
 from evalscope.utils.io_utils import OutputsStructure, yaml_to_dict
 from evalscope.utils.logger import configure_logging, get_logger
 from evalscope.version import __version__
+from .arguments import add_argument
+from .constants import DATASET_TOKEN, LATEX_DELIMITERS, MODEL_TOKEN, PLOTLY_THEME, REPORT_TOKEN
 
 logger = get_logger()
 
-PLOTLY_THEME = 'plotly_dark'
-REPORT_TOKEN = '@@'
-MODEL_TOKEN = '::'
-DATASET_TOKEN = ', '
-LATEX_DELIMITERS = [{
-    'left': '$$',
-    'right': '$$',
-    'display': True
-}, {
-    'left': '$',
-    'right': '$',
-    'display': False
-}, {
-    'left': '\\(',
-    'right': '\\)',
-    'display': False
-}, {
-    'left': '\\[',
-    'right': '\\]',
-    'display': True
-}]
-
 
 def scan_for_report_folders(root_path):
     """Scan for folders containing reports subdirectories"""
@@ -185,6 +165,13 @@ def get_single_dataset_df(df: pd.DataFrame, dataset_name: str):
     return df, styler
 
 
+def get_report_analysis(report_list: List[Report], dataset_name: str) -> str:
+    for report in report_list:
+        if report.dataset_name == dataset_name:
+            return report.analysis
+    return 'N/A'
+
+
 def plot_single_dataset_scores(df: pd.DataFrame):
     # TODO: add metric radio and relace category name
     plot = px.bar(
@@ -456,6 +443,10 @@ def create_single_model_tab(sidebar: SidebarComponents, lang: str):
             'zh': '数据集分数',
             'en': 'Dataset Scores'
         },
+        'report_analysis': {
+            'zh': '报告智能分析',
+            'en': 'Report Intelligent Analysis'
+        },
         'dataset_scores_table': {
             'zh': '数据集分数表',
             'en': 'Dataset Scores Table'
@@ -511,6 +502,9 @@ def create_single_model_tab(sidebar: SidebarComponents, lang: str):
     with gr.Tab(locale_dict['dataset_details'][lang]):
         dataset_radio = gr.Radio(
             label=locale_dict['select_dataset'][lang], choices=[], show_label=True, interactive=True)
+        # show dataset details
+        with gr.Accordion(locale_dict['report_analysis'][lang], open=True):
+            report_analysis = gr.Markdown(value='N/A', show_copy_button=True)
         gr.Markdown(f'### {locale_dict["dataset_scores"][lang]}')
         dataset_plot = gr.Plot(value=None, scale=1, label=locale_dict['dataset_scores'][lang])
         gr.Markdown(f'### {locale_dict["dataset_scores_table"][lang]}')
@@ -586,15 +580,16 @@ def create_single_model_tab(sidebar: SidebarComponents, lang: str):
     @gr.on(
         triggers=[dataset_radio.change, report_list.change],
         inputs=[dataset_radio, report_list],
-        outputs=[dataset_plot, dataset_table, subset_select, data_review_df])
+        outputs=[dataset_plot, dataset_table, subset_select, data_review_df, report_analysis])
     def update_single_report_dataset(dataset_name, report_list):
         logger.debug(f'Updating single report dataset: {dataset_name}')
         report_df = get_data_frame(report_list)
+        analysis = get_report_analysis(report_list, dataset_name)
         data_score_df, styler = get_single_dataset_df(report_df, dataset_name)
         data_score_plot = plot_single_dataset_scores(data_score_df)
         subsets = data_score_df[ReportKey.subset_name].unique().tolist()
         logger.debug(f'subsets: {subsets}')
-        return data_score_plot, styler, gr.update(choices=subsets, value=None), None
+        return data_score_plot, styler, gr.update(choices=subsets, value=None), None, analysis
 
     @gr.on(
         triggers=[subset_select.change],

evalscope/app/constants.py

@@ -0,0 +1,21 @@
+PLOTLY_THEME = 'plotly_dark'
+REPORT_TOKEN = '@@'
+MODEL_TOKEN = '::'
+DATASET_TOKEN = ', '
+LATEX_DELIMITERS = [{
+    'left': '$$',
+    'right': '$$',
+    'display': True
+}, {
+    'left': '$',
+    'right': '$',
+    'display': False
+}, {
+    'left': '\\(',
+    'right': '\\)',
+    'display': False
+}, {
+    'left': '\\[',
+    'right': '\\]',
+    'display': True
+}]

evalscope/arguments.py

@@ -67,7 +67,7 @@ def add_argument(parser: argparse.ArgumentParser):
     parser.add_argument('--eval-config', type=str, required=False, help='The eval task config file path for evaluation backend.')  # noqa: E501
     parser.add_argument('--stage', type=str, default='all', help='The stage of evaluation pipeline.',
                         choices=[EvalStage.ALL, EvalStage.INFER, EvalStage.REVIEW])
-    parser.add_argument('--limit', type=int, default=None, help='Max evaluation samples num for each subset.')
+    parser.add_argument('--limit', type=float, default=None, help='Max evaluation samples num for each subset.')
     parser.add_argument('--eval-batch-size', type=int, default=1, help='The batch size for evaluation.')
 
     # Cache and working directory arguments
@@ -89,6 +89,7 @@ def add_argument(parser: argparse.ArgumentParser):
     parser.add_argument('--judge-strategy', type=str, default=JudgeStrategy.AUTO, help='The judge strategy.')
     parser.add_argument('--judge-model-args', type=json.loads, default='{}', help='The judge model args, should be a json string.')  # noqa: E501
     parser.add_argument('--judge-worker-num', type=int, default=1, help='The number of workers for the judge model.')
+    parser.add_argument('--analysis-report', action='store_true', default=False, help='Generate analysis report for the evaluation results using judge model.')  # noqa: E501
     # yapf: enable
 
 

evalscope/backend/opencompass/backend_manager.py

@@ -1,4 +1,5 @@
 # Copyright (c) Alibaba, Inc. and its affiliates.
+import os
 import subprocess
 import tempfile
 from dataclasses import asdict
@@ -204,7 +205,7 @@ class OpenCompassBackendManager(BackendManager):
                     model_d['meta_template'] = get_template(model_d['meta_template'])
 
                 # set the 'abbr' as the 'path' if 'abbr' is not specified
-                model_d['abbr'] = model_d['path']
+                model_d['abbr'] = os.path.basename(model_d['path'])
 
                 model_config = ApiModelConfig(**model_d)
                 models.append(asdict(model_config))

evalscope/backend/rag_eval/cmteb/arguments.py

@@ -11,7 +11,9 @@ class ModelArguments:
     pooling_mode: Optional[str] = None
     max_seq_length: int = 512  # max sequence length
     # prompt for llm based model
-    prompt: str = ''
+    prompt: Optional[str] = None
+    # prompts dictionary for different tasks, if prompt is not set
+    prompts: Optional[Dict[str, str]] = None
     # model kwargs
     model_kwargs: dict = field(default_factory=dict)
     # config kwargs
@@ -33,6 +35,7 @@ class ModelArguments:
             'pooling_mode': self.pooling_mode,
             'max_seq_length': self.max_seq_length,
             'prompt': self.prompt,
+            'prompts': self.prompts,
             'model_kwargs': self.model_kwargs,
             'config_kwargs': self.config_kwargs,
             'encode_kwargs': self.encode_kwargs,

evalscope/backend/rag_eval/cmteb/task_template.py

@@ -1,6 +1,6 @@
 import mteb
 import os
-from mteb.task_selection import results_to_dataframe
+from tabulate import tabulate
 
 from evalscope.backend.rag_eval import EmbeddingModel, cmteb
 from evalscope.utils.logger import get_logger
@@ -12,14 +12,27 @@ def show_results(output_folder, model, results):
     model_name = model.mteb_model_meta.model_name_as_path()
     revision = model.mteb_model_meta.revision
 
-    results_df = results_to_dataframe({model_name: {revision: results}})
+    data = []
+    for model_res in results:
+        main_res = model_res.only_main_score()
+        for split, score in main_res.scores.items():
+            for sub_score in score:
+                data.append({
+                    'Model': model_name.replace('eval__', ''),
+                    'Revision': revision,
+                    'Task Type': main_res.task_type,
+                    'Task': main_res.task_name,
+                    'Split': split,
+                    'Subset': sub_score['hf_subset'],
+                    'Main Score': sub_score['main_score'],
+                })
 
     save_path = os.path.join(
         output_folder,
         model_name,
         revision,
     )
-    logger.info(f'Evaluation results:\n{results_df.to_markdown()}')
+    logger.info(f'Evaluation results:\n{tabulate(data, headers="keys", tablefmt="grid")}')
     logger.info(f'Evaluation results saved in {os.path.abspath(save_path)}')
 
 
@@ -34,6 +47,7 @@ def one_stage_eval(
     tasks = cmteb.TaskBase.get_tasks(task_names=eval_args['tasks'], dataset_path=custom_dataset_path)
     evaluation = mteb.MTEB(tasks=tasks)
 
+    eval_args['encode_kwargs'] = model_args.get('encode_kwargs', {})
     # run evaluation
     results = evaluation.run(model, **eval_args)
 
@@ -66,6 +80,7 @@ def two_stage_eval(
             overwrite_results=True,
             hub=eval_args['hub'],
             limits=eval_args['limits'],
+            encode_kwargs=model1_args.get('encode_kwargs', {}),
         )
         # stage 2: run cross encoder
         results = evaluation.run(
@@ -77,6 +92,7 @@ def two_stage_eval(
             overwrite_results=True,
             hub=eval_args['hub'],
             limits=eval_args['limits'],
+            encode_kwargs=model2_args.get('encode_kwargs', {}),
         )
 
         # save and log results

evalscope/backend/rag_eval/cmteb/tasks/CustomTask.py

@@ -9,7 +9,6 @@ class CustomRetrieval(AbsTaskRetrieval):
     ignore_identical_ids: bool = True
 
     def __init__(self, dataset_path: Optional[str] = 'custom_eval/text/retrieval', **kwargs):
-        super().__init__(**kwargs)
         self.metadata = TaskMetadata(
             name='CustomRetrieval',
             description='CustomRetrieval Task',
@@ -34,6 +33,7 @@ class CustomRetrieval(AbsTaskRetrieval):
             bibtex_citation='',
             descriptive_stats={},
         )
+        super().__init__(**kwargs)
 
     def load_data(self, **kwargs):
         if self.data_loaded:

evalscope/backend/rag_eval/utils/embedding.py

@@ -2,6 +2,7 @@ import os
 import torch
 from langchain_core.embeddings import Embeddings
 from langchain_openai.embeddings import OpenAIEmbeddings
+from mteb.encoder_interface import PromptType
 from sentence_transformers import models
 from sentence_transformers.cross_encoder import CrossEncoder
 from sentence_transformers.SentenceTransformer import SentenceTransformer
@@ -12,6 +13,7 @@ from typing import Dict, List, Optional, Union
 from evalscope.backend.rag_eval.utils.tools import download_model
 from evalscope.constants import HubType
 from evalscope.utils.logger import get_logger
+from evalscope.utils.utils import get_supported_params
 
 logger = get_logger()
 
@@ -22,14 +24,14 @@ class BaseModel(Embeddings):
         self,
         model_name_or_path: str = '',
         max_seq_length: int = 512,
-        prompt: str = '',
+        prompt: Optional[str] = None,
+        prompts: Optional[Dict[str, str]] = None,
         revision: Optional[str] = 'master',
         **kwargs,
     ):
         self.model_name_or_path = model_name_or_path
         self.max_seq_length = max_seq_length
         self.model_kwargs = kwargs.pop('model_kwargs', {})
-        self.model_kwargs['trust_remote_code'] = True
 
         self.config_kwargs = kwargs.pop('config_kwargs', {})
         self.config_kwargs['trust_remote_code'] = True
@@ -38,7 +40,9 @@ class BaseModel(Embeddings):
         self.encode_kwargs['convert_to_tensor'] = True
 
         self.prompt = prompt
+        self.prompts = prompts if prompts else {}
         self.revision = revision
+        self.framework = ['PyTorch']
 
     @property
     def mteb_model_meta(self):
@@ -46,10 +50,22 @@ class BaseModel(Embeddings):
         from mteb import ModelMeta
 
         return ModelMeta(
-            name=os.path.basename(self.model_name_or_path),
+            name='eval/' + os.path.basename(self.model_name_or_path),  # Ensure the name contains a slash
             revision=self.revision,
             languages=None,
             release_date=None,
+            n_parameters=None,
+            memory_usage_mb=None,
+            max_tokens=None,
+            embed_dim=None,
+            license=None,
+            open_weights=None,
+            public_training_code=None,
+            public_training_data=None,
+            similarity_fn_name=None,
+            use_instructions=None,
+            training_datasets=None,
+            framework=self.framework,
         )
 
     def embed_documents(self, texts: List[str]) -> List[List[float]]:
@@ -61,7 +77,7 @@ class BaseModel(Embeddings):
         Returns:
             List of embeddings.
         """
-        return self.encode_corpus(texts).tolist()
+        return self.encode(texts).tolist()
 
     def embed_query(self, text: str) -> List[float]:
         """Embed query text. Compact langchain.
@@ -72,19 +88,17 @@ class BaseModel(Embeddings):
         Returns:
             Embedding.
         """
-        return self.encode_queries(text).tolist()
+        return self.encode(text).tolist()
 
     def encode(self, texts: Union[str, List[str]], **kwargs) -> List[List[float]]:
         """Embed text."""
         raise NotImplementedError
 
-    def encode_queries(self, queries: List[str], **kwargs) -> list[torch.Tensor]:
-        """Embed query text. Compact mteb."""
-        raise NotImplementedError
-
-    def encode_corpus(self, corpus: Union[List[str], List[Dict[str, str]]], **kwargs) -> list[torch.Tensor]:
-        """Embed search docs . Compact mteb."""
-        raise NotImplementedError
+    def get_prompt(self, task_name: str) -> Optional[str]:
+        """Get prompt for the given task name."""
+        if self.prompt:
+            return self.prompt
+        return self.prompts.get(task_name, None)
 
 
 class SentenceTransformerModel(BaseModel):
@@ -92,6 +106,9 @@ class SentenceTransformerModel(BaseModel):
     def __init__(self, model_name_or_path: str, pooling_mode: Optional[str] = None, **kwargs):
         super().__init__(model_name_or_path, **kwargs)
 
+        self.framework = ['Sentence Transformers', 'PyTorch']
+
+        self.model_kwargs['trust_remote_code'] = True
         if not pooling_mode:
             self.model = SentenceTransformer(
                 self.model_name_or_path,
@@ -112,36 +129,47 @@ class SentenceTransformerModel(BaseModel):
 
         self.model.max_seq_length = self.max_seq_length
 
-    def encode(self, texts: Union[str, List[str]], prompt=None, **kwargs) -> List[torch.Tensor]:
-        kwargs.pop('prompt_name', '')  # remove prompt name, use prompt
+        self.supported_encode_params = get_supported_params(self.model.encode)
+
+    def encode(self, texts: Union[str, List[str]], **kwargs) -> List[torch.Tensor]:
+        # pop unused kwargs
+        extra_params = {}
+        for key in list(kwargs.keys()):
+            if key not in self.supported_encode_params:
+                extra_params[key] = kwargs.pop(key)
         self.encode_kwargs.update(kwargs)
 
+        # set prompt if provided
+        prompt = None
+        prompt_type = extra_params.pop('prompt_type', '')
+        task_name = extra_params.pop('task_name', '')
+        if prompt_type and prompt_type == PromptType.query:
+            prompt = self.get_prompt(task_name)
+
         embeddings = self.model.encode(texts, prompt=prompt, **self.encode_kwargs)
         assert isinstance(embeddings, Tensor)
         return embeddings.cpu().detach()
 
-    def encode_queries(self, queries, **kwargs):
-        return self.encode(queries, prompt=self.prompt)
-
-    def encode_corpus(self, corpus, **kwargs):
-        if isinstance(corpus[0], dict):
-            input_texts = ['{} {}'.format(doc.get('title', ''), doc['text']).strip() for doc in corpus]
-        else:
-            input_texts = corpus
-        return self.encode(input_texts)
-
 
 class CrossEncoderModel(BaseModel):
 
     def __init__(self, model_name_or_path: str, **kwargs):
         super().__init__(model_name_or_path, **kwargs)
+
+        self.framework = ['Sentence Transformers', 'PyTorch']
+
         self.model = CrossEncoder(
             self.model_name_or_path,
             trust_remote_code=True,
             max_length=self.max_seq_length,
+            automodel_args=self.model_kwargs,
         )
+        self.supported_encode_params = get_supported_params(self.model.predict)
 
     def predict(self, sentences: List[List[str]], **kwargs) -> Tensor:
+        for key in list(kwargs.keys()):
+            if key not in self.supported_encode_params:
+                kwargs.pop(key)
         self.encode_kwargs.update(kwargs)
 
         if len(sentences[0]) == 3:  # Note: For mteb retrieval task
@@ -163,6 +191,7 @@ class APIEmbeddingModel(BaseModel):
         self.openai_api_base = kwargs.get('api_base')
         self.openai_api_key = kwargs.get('api_key')
         self.dimensions = kwargs.get('dimensions')
+        self.framework = ['API']
 
         self.model = OpenAIEmbeddings(
             model=self.model_name,
@@ -175,26 +204,37 @@ class APIEmbeddingModel(BaseModel):
 
         self.batch_size = self.encode_kwargs.get('batch_size', 10)
 
+        self.supported_encode_params = get_supported_params(self.model.embed_documents)
+
     def encode(self, texts: Union[str, List[str]], **kwargs) -> Tensor:
+        # pop unused kwargs
+        extra_params = {}
+        for key in list(kwargs.keys()):
+            if key not in self.supported_encode_params:
+                extra_params[key] = kwargs.pop(key)
+        self.encode_kwargs.update(kwargs)
+
+        # set prompt if provided
+        prompt = None
+        prompt_type = extra_params.pop('prompt_type', '')
+        task_name = extra_params.pop('task_name', '')
+        if prompt_type and prompt_type == PromptType.query:
+            prompt = self.get_prompt(task_name)
+
         if isinstance(texts, str):
             texts = [texts]
 
         embeddings: List[List[float]] = []
         for i in tqdm(range(0, len(texts), self.batch_size)):
-            response = self.model.embed_documents(texts[i:i + self.batch_size], chunk_size=self.batch_size)
+            # set prompt if provided
+            if prompt is not None:
+                batch_texts = [prompt + text for text in texts[i:i + self.batch_size]]
+            else:
+                batch_texts = texts[i:i + self.batch_size]
+            response = self.model.embed_documents(batch_texts, chunk_size=self.batch_size)
             embeddings.extend(response)
         return torch.tensor(embeddings)
 
-    def encode_queries(self, queries, **kwargs):
-        return self.encode(queries, **kwargs)
-
-    def encode_corpus(self, corpus, **kwargs):
-        if isinstance(corpus[0], dict):
-            input_texts = ['{} {}'.format(doc.get('title', ''), doc['text']).strip() for doc in corpus]
-        else:
-            input_texts = corpus
-        return self.encode(input_texts, **kwargs)
-
 
 class EmbeddingModel:
     """Custom embeddings"""

evalscope/benchmarks/benchmark.py

@@ -28,6 +28,7 @@ class BenchmarkMeta:
     system_prompt: Optional[str] = None
     query_template: Optional[str] = None
     pretty_name: Optional[str] = None
+    description: Optional[str] = None
     filters: Optional[OrderedDict] = None
     extra_params: Optional[Dict] = field(default_factory=dict)