evalscope 0.13.1__py3-none-any.whl → 0.14.0__py3-none-any.whl

evalscope/arguments.py

@@ -77,7 +77,7 @@ def add_argument(parser: argparse.ArgumentParser):
     # LLMJudge arguments
     parser.add_argument('--judge-strategy', type=str, default=JudgeStrategy.AUTO, help='The judge strategy.')
     parser.add_argument('--judge-model-args', type=json.loads, default='{}', help='The judge model args, should be a json string.')  # noqa: E501
-    parser.add_argument('--judge-worker-num', type=int, default=8, help='The number of workers for the judge model.')
+    parser.add_argument('--judge-worker-num', type=int, default=1, help='The number of workers for the judge model.')
     # yapf: enable
 
 

evalscope/backend/rag_eval/__init__.py

@@ -1,4 +1,4 @@
-from evalscope.backend.rag_eval.backend_manager import RAGEvalBackendManager
+from evalscope.backend.rag_eval.backend_manager import RAGEvalBackendManager, Tools
 from evalscope.backend.rag_eval.utils.clip import VisionModel
 from evalscope.backend.rag_eval.utils.embedding import EmbeddingModel
 from evalscope.backend.rag_eval.utils.llm import LLM, ChatOpenAI, LocalLLM

evalscope/backend/rag_eval/backend_manager.py

@@ -8,6 +8,12 @@ from evalscope.utils.logger import get_logger
 logger = get_logger()
 
 
+class Tools:
+    MTEB = 'mteb'
+    RAGAS = 'ragas'
+    CLIP_BENCHMARK = 'clip_benchmark'
+
+
 class RAGEvalBackendManager(BackendManager):
 
     def __init__(self, config: Union[str, dict], **kwargs):
@@ -47,9 +53,19 @@ class RAGEvalBackendManager(BackendManager):
         from evalscope.backend.rag_eval.ragas.tasks import generate_testset
 
         if testset_args is not None:
-            generate_testset(TestsetGenerationArguments(**testset_args))
+            if isinstance(testset_args, dict):
+                generate_testset(TestsetGenerationArguments(**testset_args))
+            elif isinstance(testset_args, TestsetGenerationArguments):
+                generate_testset(testset_args)
+            else:
+                raise ValueError('Please provide the testset generation arguments.')
         if eval_args is not None:
-            rag_eval(EvaluationArguments(**eval_args))
+            if isinstance(eval_args, dict):
+                rag_eval(EvaluationArguments(**eval_args))
+            elif isinstance(eval_args, EvaluationArguments):
+                rag_eval(eval_args)
+            else:
+                raise ValueError('Please provide the evaluation arguments.')
 
     @staticmethod
     def run_clip_benchmark(args):
@@ -59,17 +75,17 @@ class RAGEvalBackendManager(BackendManager):
 
     def run(self, *args, **kwargs):
         tool = self.config_d.pop('tool')
-        if tool.lower() == 'mteb':
+        if tool.lower() == Tools.MTEB:
             self._check_env('mteb')
             model_args = self.config_d['model']
             eval_args = self.config_d['eval']
             self.run_mteb(model_args, eval_args)
-        elif tool.lower() == 'ragas':
+        elif tool.lower() == Tools.RAGAS:
             self._check_env('ragas')
             testset_args = self.config_d.get('testset_generation', None)
             eval_args = self.config_d.get('eval', None)
             self.run_ragas(testset_args, eval_args)
-        elif tool.lower() == 'clip_benchmark':
+        elif tool.lower() == Tools.CLIP_BENCHMARK:
             self._check_env('webdataset')
             self.run_clip_benchmark(self.config_d['eval'])
         else:

evalscope/backend/rag_eval/cmteb/arguments.py

@@ -20,6 +20,12 @@ class ModelArguments:
     encode_kwargs: dict = field(default_factory=lambda: {'show_progress_bar': True, 'batch_size': 32})
     hub: str = 'modelscope'  # modelscope or huggingface
 
+    # for API embedding model
+    model_name: Optional[str] = None
+    api_base: Optional[str] = None
+    api_key: Optional[str] = None
+    dimensions: Optional[int] = None
+
     def to_dict(self) -> Dict[str, Any]:
         return {
             'model_name_or_path': self.model_name_or_path,
@@ -31,6 +37,10 @@ class ModelArguments:
             'config_kwargs': self.config_kwargs,
             'encode_kwargs': self.encode_kwargs,
             'hub': self.hub,
+            'model_name': self.model_name,
+            'api_base': self.api_base,
+            'api_key': self.api_key,
+            'dimensions': self.dimensions,
         }
 
 

evalscope/backend/rag_eval/ragas/arguments.py

@@ -21,7 +21,6 @@ class TestsetGenerationArguments:
     """
     generator_llm: Dict = field(default_factory=dict)
     embeddings: Dict = field(default_factory=dict)
-    distribution: str = field(default_factory=lambda: {'simple': 0.5, 'multi_context': 0.4, 'reasoning': 0.1})
     # For LLM based evaluation
     # available: ['english', 'hindi', 'marathi', 'chinese', 'spanish', 'amharic', 'arabic',
     # 'armenian', 'bulgarian', 'urdu', 'russian', 'polish', 'persian', 'dutch', 'danish',

evalscope/backend/rag_eval/ragas/tasks/testset_generation.py

@@ -67,9 +67,14 @@ def get_persona(llm, kg, language):
 
 
 def load_data(file_path):
-    from langchain_community.document_loaders import UnstructuredFileLoader
+    import nltk
+    from langchain_unstructured import UnstructuredLoader
 
-    loader = UnstructuredFileLoader(file_path, mode='single')
+    if nltk.data.find('taggers/averaged_perceptron_tagger_eng') is False:
+        # need to download nltk data for the first time
+        nltk.download('averaged_perceptron_tagger_eng')
+
+    loader = UnstructuredLoader(file_path)
     data = loader.load()
     return data
 

evalscope/backend/rag_eval/ragas/tasks/translate_prompt.py

@@ -2,7 +2,6 @@ import asyncio
 import os
 from ragas.llms import BaseRagasLLM
 from ragas.prompt import PromptMixin, PydanticPrompt
-from ragas.utils import RAGAS_SUPPORTED_LANGUAGE_CODES
 from typing import List
 
 from evalscope.utils.logger import get_logger
@@ -16,10 +15,6 @@ async def translate_prompt(
     llm: BaseRagasLLM,
     adapt_instruction: bool = False,
 ):
-    if target_lang not in RAGAS_SUPPORTED_LANGUAGE_CODES:
-        logger.warning(f'{target_lang} is not in supported language: {list(RAGAS_SUPPORTED_LANGUAGE_CODES)}')
-        return
-
     if not issubclass(type(prompt_user), PromptMixin):
         logger.info(f"{prompt_user} is not a PromptMixin, don't translate it")
         return

evalscope/backend/rag_eval/utils/embedding.py

@@ -1,10 +1,12 @@
 import os
 import torch
 from langchain_core.embeddings import Embeddings
+from langchain_openai.embeddings import OpenAIEmbeddings
 from sentence_transformers import models
 from sentence_transformers.cross_encoder import CrossEncoder
 from sentence_transformers.SentenceTransformer import SentenceTransformer
 from torch import Tensor
+from tqdm import tqdm
 from typing import Dict, List, Optional, Union
 
 from evalscope.backend.rag_eval.utils.tools import download_model
@@ -18,10 +20,10 @@ class BaseModel(Embeddings):
 
     def __init__(
         self,
-        model_name_or_path: str,
+        model_name_or_path: str = '',
         max_seq_length: int = 512,
         prompt: str = '',
-        revision: Optional[str] = None,
+        revision: Optional[str] = 'master',
         **kwargs,
     ):
         self.model_name_or_path = model_name_or_path
@@ -139,7 +141,7 @@ class CrossEncoderModel(BaseModel):
             max_length=self.max_seq_length,
         )
 
-    def predict(self, sentences: List[List[str]], **kwargs) -> List[List[float]]:
+    def predict(self, sentences: List[List[str]], **kwargs) -> Tensor:
         self.encode_kwargs.update(kwargs)
 
         if len(sentences[0]) == 3:  # Note: For mteb retrieval task
@@ -154,6 +156,46 @@ class CrossEncoderModel(BaseModel):
         return embeddings
 
 
+class APIEmbeddingModel(BaseModel):
+
+    def __init__(self, **kwargs):
+        self.model_name = kwargs.get('model_name')
+        self.openai_api_base = kwargs.get('api_base')
+        self.openai_api_key = kwargs.get('api_key')
+        self.dimensions = kwargs.get('dimensions')
+
+        self.model = OpenAIEmbeddings(
+            model=self.model_name,
+            openai_api_base=self.openai_api_base,
+            openai_api_key=self.openai_api_key,
+            dimensions=self.dimensions,
+            check_embedding_ctx_length=False)
+
+        super().__init__(model_name_or_path=self.model_name, **kwargs)
+
+        self.batch_size = self.encode_kwargs.get('batch_size', 10)
+
+    def encode(self, texts: Union[str, List[str]], **kwargs) -> Tensor:
+        if isinstance(texts, str):
+            texts = [texts]
+
+        embeddings: List[List[float]] = []
+        for i in tqdm(range(0, len(texts), self.batch_size)):
+            response = self.model.embed_documents(texts[i:i + self.batch_size], chunk_size=self.batch_size)
+            embeddings.extend(response)
+        return torch.tensor(embeddings)
+
+    def encode_queries(self, queries, **kwargs):
+        return self.encode(queries, **kwargs)
+
+    def encode_corpus(self, corpus, **kwargs):
+        if isinstance(corpus[0], dict):
+            input_texts = ['{} {}'.format(doc.get('title', ''), doc['text']).strip() for doc in corpus]
+        else:
+            input_texts = corpus
+        return self.encode(input_texts, **kwargs)
+
+
 class EmbeddingModel:
     """Custom embeddings"""
 
@@ -165,6 +207,10 @@ class EmbeddingModel:
         revision: Optional[str] = 'master',
         **kwargs,
     ):
+        if kwargs.get('model_name'):
+            # If model_name is provided, use OpenAIEmbeddings
+            return APIEmbeddingModel(**kwargs)
+
         # If model path does not exist and hub is 'modelscope', download the model
         if not os.path.exists(model_name_or_path) and hub == HubType.MODELSCOPE:
             model_name_or_path = download_model(model_name_or_path, revision)

evalscope/backend/rag_eval/utils/llm.py

@@ -2,11 +2,11 @@ import os
 from langchain_core.callbacks.manager import CallbackManagerForLLMRun
 from langchain_core.language_models.llms import LLM as BaseLLM
 from langchain_openai import ChatOpenAI
-from modelscope.utils.hf_util import GenerationConfig
+from transformers.generation.configuration_utils import GenerationConfig
 from typing import Any, Dict, Iterator, List, Mapping, Optional
 
 from evalscope.constants import DEFAULT_MODEL_REVISION
-from evalscope.models import ChatGenerationModelAdapter
+from evalscope.models import ChatGenerationModelAdapter, LocalModel
 
 
 class LLM:
@@ -16,9 +16,9 @@ class LLM:
         api_base = kw.get('api_base', None)
         if api_base:
             return ChatOpenAI(
-                model_name=kw.get('model_name', ''),
-                openai_api_base=api_base,
-                openai_api_key=kw.get('api_key', 'EMPTY'),
+                model=kw.get('model_name', ''),
+                base_url=api_base,
+                api_key=kw.get('api_key', 'EMPTY'),
             )
         else:
             return LocalLLM(**kw)
@@ -38,8 +38,7 @@ class LocalLLM(BaseLLM):
         super().__init__(**kw)
         self.model_name = os.path.basename(self.model_name_or_path)
         self.model = ChatGenerationModelAdapter(
-            model_id=self.model_name_or_path,
-            model_revision=self.model_revision,
+            model=LocalModel(model_id=self.model_name_or_path, model_revision=self.model_revision),
             generation_config=GenerationConfig(**self.generation_config) if self.generation_config else None,
         )
 
@@ -53,8 +52,8 @@ class LocalLLM(BaseLLM):
         """Run the LLM on the given input."""
         infer_cfg = {'stop': stop}
 
-        response = self.model._model_generate(prompt, infer_cfg)
-        return response
+        response, _ = self.model._model_generate([prompt], infer_cfg=infer_cfg)
+        return response[0][0]
 
     @property
     def _identifying_params(self) -> Dict[str, Any]:

evalscope/backend/vlm_eval_kit/backend_manager.py

@@ -1,4 +1,5 @@
 import copy
+import os
 import subprocess
 from functools import partial
 from typing import Optional, Union
@@ -66,8 +67,9 @@ class VLMEvalKitBackendManager(BackendManager):
                     del remain_cfg['name']  # remove not used args
                     del remain_cfg['type']  # remove not used args
 
-                    self.valid_models.update({model_type: partial(model_class, model=model_type, **remain_cfg)})
-                    new_model_names.append(model_type)
+                    norm_model_type = os.path.basename(model_type).replace(':', '-').replace('.', '_')
+                    self.valid_models.update({norm_model_type: partial(model_class, model=model_type, **remain_cfg)})
+                    new_model_names.append(norm_model_type)
                 else:
                     remain_cfg = copy.deepcopy(model_cfg)
                     del remain_cfg['name']  # remove not used args

evalscope/benchmarks/alpaca_eval/alpaca_eval_adapter.py

@@ -0,0 +1,109 @@
+import re
+from collections import defaultdict
+from typing import Any, List
+
+from evalscope.benchmarks import Benchmark, DataAdapter
+from evalscope.metrics import Metric, mean, metric_registry
+from evalscope.metrics.llm_judge import LLMJudge
+from evalscope.utils.logger import get_logger
+
+# flake8: noqa
+
+logger = get_logger()
+
+GRADER_SYSTEM_PROMPT = """You are a highly efficient assistant, who evaluates and selects the best large language model (LLMs) based on the quality of their responses to a given instruction. This process will be used to create a leaderboard reflecting the most accurate and human-preferred answers."""
+
+GRADER_TEMPLATE = """
+I require a leaderboard for various large language models. I'll provide you with prompts given to these models and their corresponding outputs. Your task is to assess these responses, and select the model that produces the best output from a human perspective.
+
+## Instruction
+
+{{
+    "instruction": "{instruction}"
+}}
+
+## Model Outputs
+
+Here are the unordered outputs from the models. Each output is associated with a specific model, identified by a unique model identifier.
+
+{{
+    {{
+        "model_identifier": "m",
+        "output": "{output_1}"
+    }},
+    {{
+        "model_identifier": "M",
+        "output": "{output_2}"
+    }}
+}}
+
+## Task
+
+Evaluate the models based on the quality and relevance of their outputs, and select the model that generated the best output. Answer by providing the model identifier of the best model. We will use your output as the name of the best model, so make sure your output only contains one of the following model identifiers and nothing else (no quotes, no spaces, no new lines, ...): m or M.
+
+## Best Model Identifier
+""".strip()  # noqa: E501
+
+
+@Benchmark.register(
+    name='alpaca_eval',
+    pretty_name='AlpacaEval2.0',
+    dataset_id='AI-ModelScope/alpaca_eval',
+    subset_list=['alpaca_eval_gpt4_baseline'],
+    metric_list=['winrate'],
+    few_shot_num=0,
+    train_split=None,
+    eval_split='eval')
+class AlpacaEvalAdapter(DataAdapter):
+
+    def __init__(self, *args, **kwargs):
+        super().__init__(*args, **kwargs)
+
+        # register metrics
+        metric_registry.register(Metric(name='winrate', object=mean))
+
+        # whether to use LLM as a judge
+        self.llm_as_a_judge = True
+
+    def gen_prompt(self, input_d: dict, subset_name: str, few_shot_list: list, **kwargs) -> dict:
+        question = input_d['instruction']
+        return self.gen_prompt_data(question)
+
+    def get_gold_answer(self, input_d: dict) -> str:
+        return input_d['output']
+
+    def parse_pred_result(self, result: str, raw_input_d: dict = None, **kwargs) -> str:
+        return result.strip()
+
+    def match(self, gold: str, pred: str):
+        # simple match
+        logger.warning(f'Please use LLMJudge to match the result for {self.name}')
+        return None
+
+    def llm_match(self, gold: Any, pred: Any, judge: LLMJudge, **kwargs) -> bool:
+        raw_input = kwargs.get('raw_input', None)
+        instruction = raw_input['instruction']
+        # gold is baseline answer 'm', pred is model answer 'M'
+        prompt = GRADER_TEMPLATE.format(instruction=instruction, output_1=gold, output_2=pred)
+        # get grading response
+        grading_response = judge(prompt, system_prompt=GRADER_SYSTEM_PROMPT)
+        # parse grading response
+        match = re.search(r'(m|M)', grading_response)
+        res = match.group(0) if match else None
+        if res:
+            return res == 'M'
+        else:
+            logger.info(f'Failed to parse grading response: {prompt=}\n {grading_response=}')
+            return None
+
+    def compute_metric(self, review_res_list: List[bool], **kwargs) -> List[dict]:
+        """
+        compute weighted mean of the bleu score of all samples
+
+        Args:
+            review_res_list: [{'is_correct': 1, 'is_incorrect': 0, 'is_not_attempted': 0}, ...]
+        """
+        # zip dict answers
+        res_list = [res for res in review_res_list if res is not None]
+
+        return super().compute_metric(res_list, **kwargs)

evalscope/benchmarks/arc/arc_adapter.py

@@ -134,7 +134,7 @@ class ARCAdapter(DataAdapter):
         if self.model_adapter == OutputType.MULTIPLE_CHOICE:
             return result
         else:
-            return ResponseParser.parse_first_option(text=result)
+            return ResponseParser.parse_first_option(text=result, options=self.choices)
 
     def match(self, gold: str, pred: str) -> float:
         return exact_match(gold=gold, pred=pred)

evalscope/benchmarks/arena_hard/arena_hard_adapter.py

@@ -0,0 +1,120 @@
+import re
+from collections import defaultdict
+from typing import Any, List
+
+from evalscope.benchmarks import Benchmark, DataAdapter
+from evalscope.constants import AnswerKeys
+from evalscope.metrics import Metric, mean, metric_registry
+from evalscope.metrics.llm_judge import LLMJudge
+from evalscope.utils.logger import get_logger
+
+# flake8: noqa
+
+logger = get_logger()
+
+GRADER_SYSTEM_PROMPT = "Please act as an impartial judge and evaluate the quality of the responses provided by two AI assistants to the user prompt displayed below. You will be given assistant A's answer and assistant B's answer. Your job is to evaluate which assistant's answer is better.\n\nBegin your evaluation by generating your own answer to the prompt. You must provide your answers before judging any answers.\n\nWhen evaluating the assistants' answers, compare both assistants' answers with your answer. You must identify and correct any mistakes or inaccurate information.\n\nThen consider if the assistant's answers are helpful, relevant, and concise. Helpful means the answer correctly responds to the prompt or follows the instructions. Note when user prompt has any ambiguity or more than one interpretation, it is more helpful and appropriate to ask for clarifications or more information from the user than providing an answer based on assumptions. Relevant means all parts of the response closely connect or are appropriate to what is being asked. Concise means the response is clear and not verbose or excessive.\n\nThen consider the creativity and novelty of the assistant's answers when needed. Finally, identify any missing important information in the assistants' answers that would be beneficial to include when responding to the user prompt.\n\nAfter providing your explanation, you must output only one of the following choices as your final verdict with a label:\n\n1. Assistant A is significantly better: [[A>>B]]\n2. Assistant A is slightly better: [[A>B]]\n3. Tie, relatively the same: [[A=B]]\n4. Assistant B is slightly better: [[B>A]]\n5. Assistant B is significantly better: [[B>>A]]\n\nExample output: \"My final verdict is tie: [[A=B]]\"."  # noqa: E501
+
+GRADER_TEMPLATE = "<|User Prompt|>\n{question}\n\n<|The Start of Assistant A's Answer|>\n{answer_1}\n<|The End of Assistant A's Answer|>\n\n<|The Start of Assistant B's Answer|>\n{answer_2}\n<|The End of Assistant B's Answer|>".strip(
+)  # noqa: E501
+
+
+@Benchmark.register(
+    name='arena_hard',
+    pretty_name='ArenaHard',
+    dataset_id='AI-ModelScope/arena-hard-auto-v0.1',
+    metric_list=['winrate'],
+    few_shot_num=0,
+    train_split=None,
+    eval_split='test')
+class AlpacaEvalAdapter(DataAdapter):
+
+    def __init__(self, *args, **kwargs):
+        super().__init__(*args, **kwargs)
+
+        # register metrics
+        metric_registry.register(Metric(name='winrate', object=mean))
+
+        # whether to use LLM as a judge
+        self.llm_as_a_judge = True
+
+    def gen_prompt(self, input_d: dict, subset_name: str, few_shot_list: list, **kwargs) -> dict:
+        question = input_d['question']
+        return self.gen_prompt_data(question)
+
+    def get_gold_answer(self, input_d: dict) -> str:
+        return input_d['prediction']
+
+    def parse_pred_result(self, result: str, raw_input_d: dict = None, **kwargs) -> str:
+        return result.strip()
+
+    def match(self, gold: str, pred: str):
+        # simple match
+        logger.warning(f'Please use LLMJudge to match the result for {self.name}')
+        return None
+
+    def llm_match(self, gold: Any, pred: Any, judge: LLMJudge, **kwargs) -> dict:
+        from .utils import post_process_arenahard
+
+        raw_input = kwargs.get('raw_input', None)
+        question = raw_input['question']
+        # gold is baseline answer 'A', pred is model answer 'B'
+        prompt1 = GRADER_TEMPLATE.format(question=question, answer_1=gold, answer_2=pred)
+        # reverse the order
+        prompt2 = GRADER_TEMPLATE.format(question=question, answer_1=pred, answer_2=gold)
+        # get grading response
+        game1_response = judge(prompt1, system_prompt=GRADER_SYSTEM_PROMPT)
+        game2_response = judge(prompt2, system_prompt=GRADER_SYSTEM_PROMPT)
+        # parse grading response
+        res1 = post_process_arenahard(game1_response)
+        res2 = post_process_arenahard(game2_response)
+        return {
+            'model_a':
+            'gpt4-0314',
+            'model_b':
+            'test_model',
+            'games': [
+                {
+                    'user_prompt': prompt1,
+                    'judgment': game1_response,
+                    'score': res1
+                },
+                {
+                    'user_prompt': prompt2,
+                    'judgment': game2_response,
+                    'score': res2
+                },
+            ]
+        }
+
+    def compute_metric(self, review_res_list: List[dict], **kwargs) -> List[dict]:
+        """
+        compute score of the model
+        """
+        import pandas as pd
+
+        from .utils import compute_mle_elo, get_battles_from_row, get_bootstrap_result, get_win_rate_column
+
+        if isinstance(review_res_list[0], list):
+            review_res_list = [item for sublist in review_res_list for item in sublist]
+
+        battles = pd.concat([get_battles_from_row(res) for res in review_res_list])
+
+        bootstrap_online_elo = compute_mle_elo(battles)
+
+        # bootstrap_elo_lu = get_bootstrap_result(battles, compute_mle_elo, 100)
+        stats = pd.DataFrame()
+        stats['results'] = None
+        stats['results'] = stats['results'].astype('object')
+
+        for i, model in enumerate(bootstrap_online_elo.index):
+            # assert model in bootstrap_elo_lu.columns
+            stats.at[i, 'model'] = model
+            stats.at[i, 'score'] = bootstrap_online_elo[model]
+            # stats.at[i, "lower"] = np.percentile(bootstrap_elo_lu[model], 2.5)
+            # stats.at[i, "upper"] = np.percentile(bootstrap_elo_lu[model], 97.5)
+
+        # stats['score'] = get_win_rate_column(stats, 'score', 'gpt4-0314').tolist()
+
+        score = get_win_rate_column(stats, 'score', 'gpt4-0314').at['test_model']
+
+        return [{'metric_name': 'winrate', 'score': score, 'num': len(review_res_list)}]