evalscope 0.17.1__py3-none-any.whl → 1.0.1__py3-none-any.whl

evalscope/backend/rag_eval/ragas/tasks/build_distribution.py

@@ -27,7 +27,8 @@ def default_query_distribution(llm: BaseRagasLLM, kg: KnowledgeGraph, language:
             target_lang=language,
             llm=llm,
             adapt_instruction=True,
-        ))
+        )
+    )
 
     default_queries = [
         single_hop,

evalscope/backend/rag_eval/ragas/tasks/build_transform.py

@@ -44,8 +44,9 @@ def default_transforms(
         return bins
 
     def filter_doc_with_num_tokens(node, min_num_tokens=500):
-        return (node.type == NodeType.DOCUMENT
-                and num_tokens_from_string(node.properties['page_content']) > min_num_tokens)
+        return (
+            node.type == NodeType.DOCUMENT and num_tokens_from_string(node.properties['page_content']) > min_num_tokens
+        )
 
     def filter_docs(node):
         return node.type == NodeType.DOCUMENT
@@ -90,7 +91,8 @@ def default_transforms(
                 target_lang=language,
                 llm=llm,
                 adapt_instruction=True,
-            ))
+            )
+        )
 
         transforms = [
             headline_extractor,
@@ -121,7 +123,8 @@ def default_transforms(
                 target_lang=language,
                 llm=llm,
                 adapt_instruction=True,
-            ))
+            )
+        )
 
         transforms = [
             summary_extractor,

evalscope/backend/rag_eval/ragas/tasks/testset_generation.py

@@ -113,7 +113,8 @@ def generate_testset(args: TestsetGenerationArguments) -> None:
 
     # generate testset
     generator = TestsetGenerator(
-        llm=wrapped_llm, embedding_model=wrapped_embeddings, knowledge_graph=knowledge_graph, persona_list=persona_list)
+        llm=wrapped_llm, embedding_model=wrapped_embeddings, knowledge_graph=knowledge_graph, persona_list=persona_list
+    )
 
     testset = generator.generate(
         testset_size=args.test_size,

evalscope/backend/rag_eval/ragas/tasks/translate_prompt.py

@@ -34,7 +34,8 @@ async def translate_prompt(
 
     logger.info(f'Translating prompts to {target_lang}')
     adapted_prompts = await prompt_user.adapt_prompts(
-        language=target_lang, llm=llm, adapt_instruction=adapt_instruction)
+        language=target_lang, llm=llm, adapt_instruction=adapt_instruction
+    )
     prompt_user.set_prompts(**adapted_prompts)
     try:
         prompt_user.save_prompts(prompt_dir)

evalscope/backend/rag_eval/utils/embedding.py

@@ -164,6 +164,13 @@ class CrossEncoderModel(BaseModel):
             max_length=self.max_seq_length,
             automodel_args=self.model_kwargs,
         )
+        self.tokenizer = self.model.tokenizer
+        # set pad token
+        if self.tokenizer.pad_token is None:
+            self.tokenizer.pad_token = self.tokenizer.eos_token
+        if ('pad_token_id' not in self.model.config) or (self.model.config.pad_token_id is None):
+            self.model.config.update({'pad_token_id': self.tokenizer.eos_token_id})
+
         self.supported_encode_params = get_supported_params(self.model.predict)
 
     def predict(self, sentences: List[List[str]], **kwargs) -> Tensor:
@@ -189,6 +196,7 @@ class APIEmbeddingModel(BaseModel):
         self.openai_api_base = kwargs.get('api_base')
         self.openai_api_key = kwargs.get('api_key')
         self.dimensions = kwargs.get('dimensions')
+        self.check_embedding_ctx_length = kwargs.get('check_embedding_ctx_length', False)
         self.framework = ['API']
 
         self.model = OpenAIEmbeddings(
@@ -196,7 +204,8 @@ class APIEmbeddingModel(BaseModel):
             openai_api_base=self.openai_api_base,
             openai_api_key=self.openai_api_key,
             dimensions=self.dimensions,
-            check_embedding_ctx_length=False)
+            check_embedding_ctx_length=self.check_embedding_ctx_length,
+        )
 
         super().__init__(model_name_or_path=self.model_name, **kwargs)
 

evalscope/backend/rag_eval/utils/llm.py

@@ -2,11 +2,10 @@ import os
 from langchain_core.callbacks.manager import CallbackManagerForLLMRun
 from langchain_core.language_models.llms import LLM as BaseLLM
 from langchain_openai import ChatOpenAI
-from transformers.generation.configuration_utils import GenerationConfig
 from typing import Any, Dict, Iterator, List, Mapping, Optional
 
-from evalscope.constants import DEFAULT_MODEL_REVISION
-from evalscope.models import ChatGenerationModelAdapter, LocalModel
+from evalscope.api.model import GenerateConfig, Model, get_model
+from evalscope.constants import DEFAULT_MODEL_REVISION, EvalType
 
 
 class LLM:
@@ -30,16 +29,19 @@ class LocalLLM(BaseLLM):
     model_name_or_path: str
     model_revision: str = DEFAULT_MODEL_REVISION
     template_type: Optional[str] = None
-    model_name: Optional[str]
-    model: Optional[ChatGenerationModelAdapter]
-    generation_config: Optional[Dict]
+    model_name: Optional[str] = None
+    model: Optional[Model] = None
+    generation_config: Optional[Dict] = {}
 
     def __init__(self, **kw):
         super().__init__(**kw)
         self.model_name = os.path.basename(self.model_name_or_path)
-        self.model = ChatGenerationModelAdapter(
-            model=LocalModel(model_id=self.model_name_or_path, model_revision=self.model_revision),
-            generation_config=GenerationConfig(**self.generation_config) if self.generation_config else None,
+
+        # Create and initialize the local model
+        self.model = get_model(
+            model=self.model_name_or_path,
+            eval_type=EvalType.CHECKPOINT,
+            config=GenerateConfig(**self.generation_config),
         )
 
     def _call(
@@ -50,10 +52,9 @@ class LocalLLM(BaseLLM):
         **kwargs: Any,
     ) -> str:
         """Run the LLM on the given input."""
-        infer_cfg = {'stop': stop}
 
-        response, _ = self.model.predict([{'data': [prompt]}], infer_cfg=infer_cfg)
-        return response[0][0]
+        response = self.model.generate(input=prompt)
+        return response.completion
 
     @property
     def _identifying_params(self) -> Dict[str, Any]:

evalscope/benchmarks/__init__.py

@@ -4,8 +4,6 @@ import importlib
 import os
 import time
 
-from evalscope.benchmarks.benchmark import Benchmark, BenchmarkMeta
-from evalscope.benchmarks.data_adapter import DataAdapter
 from evalscope.utils import get_logger
 
 logger = get_logger()

evalscope/benchmarks/aime/aime24_adapter.py

@@ -1,5 +1,12 @@
-from evalscope.benchmarks import Benchmark, DataAdapter
-from evalscope.metrics import extract_answer, math_equal, strip_answer_string
+# Copyright (c) Alibaba, Inc. and its affiliates.
+
+from typing import Any, Dict
+
+from evalscope.api.benchmark import BenchmarkMeta, DefaultDataAdapter
+from evalscope.api.dataset import Sample
+from evalscope.api.evaluator import TaskState
+from evalscope.api.registry import register_benchmark
+from evalscope.constants import Tags
 from evalscope.utils.logger import get_logger
 
 # flake8: noqa
@@ -7,46 +14,37 @@ from evalscope.utils.logger import get_logger
 logger = get_logger()
 
 
-@Benchmark.register(
-    name='aime24',
-    pretty_name='AIME-2024',
-    tags=['Mathematics'],
-    description=
-    'The AIME 2024 benchmark is based on problems from the American Invitational Mathematics Examination, a prestigious high school mathematics competition. This benchmark tests a model’s ability to solve challenging mathematics problems by generating step-by-step solutions and providing the correct final answer.',  # noqa: E501
-    dataset_id='HuggingFaceH4/aime_2024',
-    subset_list=['default'],
-    metric_list=['AveragePass@1'],
-    few_shot_num=0,
-    train_split=None,
-    eval_split='train',  # Only train set is available
-    prompt_template='{query}\nPlease reason step by step, and put your final answer within \\boxed{{}}.',
+@register_benchmark(
+    BenchmarkMeta(
+        name='aime24',
+        pretty_name='AIME-2024',
+        tags=[Tags.MATH, Tags.REASONING],
+        description=
+        'The AIME 2024 benchmark is based on problems from the American Invitational Mathematics Examination, a prestigious high school mathematics competition. This benchmark tests a model\'s ability to solve challenging mathematics problems by generating step-by-step solutions and providing the correct final answer.',  # noqa: E501
+        dataset_id='HuggingFaceH4/aime_2024',
+        subset_list=['default'],
+        metric_list=[{
+            'acc': {
+                'numeric': True
+            }
+        }],
+        few_shot_num=0,
+        train_split=None,
+        eval_split='train',  # Only train set is available
+        prompt_template='{question}\nPlease reason step by step, and put your final answer within \\boxed{{}}.',
+    )
 )
-class AIME24Adapter(DataAdapter):
+class AIME24Adapter(DefaultDataAdapter):
 
     def __init__(self, *args, **kwargs):
         super().__init__(*args, **kwargs)
 
-    def gen_prompt(self, input_d: dict, few_shot_list: list, **kwargs) -> dict:
-        """
-        Generate the prompt for the model input.
-        """
-        problem = input_d['problem']
-        full_prompt = self.prompt_template.format(query=problem)
-
-        return self.gen_prompt_data(full_prompt)
-
-    def get_gold_answer(self, input_d: dict) -> str:
-        # Extract the gold answer from the input dict.
-        return strip_answer_string(input_d['answer'])
-
-    def parse_pred_result(self, result: str, raw_input_d: dict = None, eval_type: str = 'checkpoint') -> str:
-        """
-        Parse the model output to get the answer. Could be the best choice index.
-        """
-        # Note: Use same extraction method for both of checkpoint/service/custom
-        result = strip_answer_string(extract_answer(result))
-        return result
-
-    def match(self, gold: str, pred: str) -> float:
-        res = math_equal(pred, gold)
-        return 1.0 if res else 0.0
+    def record_to_sample(self, record: Dict[str, Any]) -> Sample:
+        return Sample(
+            input=record['problem'],
+            target=record['answer'],
+            metadata={
+                'problem_id': record.get('id', ''),
+                'solution': record.get('solution', ''),
+            },
+        )

evalscope/benchmarks/aime/aime25_adapter.py

@@ -1,5 +1,12 @@
-from evalscope.benchmarks import Benchmark, DataAdapter
-from evalscope.metrics import extract_answer, math_equal, strip_answer_string
+# Copyright (c) Alibaba, Inc. and its affiliates.
+
+from typing import Any, Dict
+
+from evalscope.api.benchmark import BenchmarkMeta, DefaultDataAdapter
+from evalscope.api.dataset import Sample
+from evalscope.api.evaluator import TaskState
+from evalscope.api.registry import register_benchmark
+from evalscope.constants import Tags
 from evalscope.utils.logger import get_logger
 
 # flake8: noqa
@@ -7,46 +14,33 @@ from evalscope.utils.logger import get_logger
 logger = get_logger()
 
 
-@Benchmark.register(
-    name='aime25',
-    pretty_name='AIME-2025',
-    tags=['Mathematics'],
-    description=
-    'The AIME 2025 benchmark is based on problems from the American Invitational Mathematics Examination, a prestigious high school mathematics competition. This benchmark tests a model’s ability to solve challenging mathematics problems by generating step-by-step solutions and providing the correct final answer.',
-    dataset_id='opencompass/AIME2025',
-    subset_list=['AIME2025-I', 'AIME2025-II'],
-    metric_list=['AveragePass@1'],
-    few_shot_num=0,
-    train_split=None,
-    eval_split='test',  # Only train set is available
-    prompt_template='{query}\nPlease reason step by step, and put your final answer within \\boxed{{}}.',
+@register_benchmark(
+    BenchmarkMeta(
+        name='aime25',
+        pretty_name='AIME-2025',
+        tags=[Tags.MATH, Tags.REASONING],
+        description=
+        'The AIME 2025 benchmark is based on problems from the American Invitational Mathematics Examination, a prestigious high school mathematics competition. This benchmark tests a model\'s ability to solve challenging mathematics problems by generating step-by-step solutions and providing the correct final answer.',
+        dataset_id='opencompass/AIME2025',
+        subset_list=['AIME2025-I', 'AIME2025-II'],
+        metric_list=[{
+            'acc': {
+                'numeric': True
+            }
+        }],
+        few_shot_num=0,
+        train_split=None,
+        eval_split='test',
+        prompt_template='{question}\nPlease reason step by step, and put your final answer within \\boxed{{}}.',
+    )
 )
-class AIME25Adapter(DataAdapter):
+class AIME25Adapter(DefaultDataAdapter):
 
     def __init__(self, *args, **kwargs):
         super().__init__(*args, **kwargs)
 
-    def gen_prompt(self, input_d: dict, few_shot_list: list, **kwargs) -> dict:
-        """
-        Generate the prompt for the model input.
-        """
-        problem = input_d['question']
-        full_prompt = self.prompt_template.format(query=problem)
-
-        return self.gen_prompt_data(full_prompt)
-
-    def get_gold_answer(self, input_d: dict) -> str:
-        # Extract the gold answer from the input dict.
-        return strip_answer_string(input_d['answer'])
-
-    def parse_pred_result(self, result: str, raw_input_d: dict = None, eval_type: str = 'checkpoint') -> str:
-        """
-        Parse the model output to get the answer. Could be the best choice index.
-        """
-        # Note: Use same extraction method for both of checkpoint/service/custom
-        result = strip_answer_string(extract_answer(result))
-        return result
-
-    def match(self, gold: str, pred: str) -> float:
-        res = math_equal(pred, gold)
-        return 1.0 if res else 0.0
+    def record_to_sample(self, record: Dict[str, Any]) -> Sample:
+        return Sample(
+            input=record['question'],
+            target=record['answer'],
+        )

evalscope/benchmarks/alpaca_eval/alpaca_eval_adapter.py

@@ -1,16 +1,17 @@
 import re
-from collections import defaultdict
-from typing import Any, List
-
-from evalscope.benchmarks import Benchmark, DataAdapter
-from evalscope.metrics import LLMJudge, Metric, mean, metric_registry
+from typing import Any, Dict
+
+from evalscope.api.benchmark import BenchmarkMeta, DefaultDataAdapter
+from evalscope.api.dataset import Sample
+from evalscope.api.evaluator import TaskState
+from evalscope.api.metric import Score
+from evalscope.api.registry import register_benchmark
+from evalscope.constants import Tags
 from evalscope.utils.logger import get_logger
 
-# flake8: noqa
-
 logger = get_logger()
 
-GRADER_SYSTEM_PROMPT = """You are a highly efficient assistant, who evaluates and selects the best large language model (LLMs) based on the quality of their responses to a given instruction. This process will be used to create a leaderboard reflecting the most accurate and human-preferred answers."""
+GRADER_SYSTEM_PROMPT = """You are a highly efficient assistant, who evaluates and selects the best large language model (LLMs) based on the quality of their responses to a given instruction. This process will be used to create a leaderboard reflecting the most accurate and human-preferred answers."""  # noqa: E501
 
 GRADER_TEMPLATE = """
 I require a leaderboard for various large language models. I'll provide you with prompts given to these models and their corresponding outputs. Your task is to assess these responses, and select the model that produces the best output from a human perspective.
@@ -44,64 +45,89 @@ Evaluate the models based on the quality and relevance of their outputs, and sel
 """.strip()  # noqa: E501
 
 
-@Benchmark.register(
-    name='alpaca_eval',
-    pretty_name='AlpacaEval2.0',
-    tags=['Instruction-Following', 'Arena'],
-    description='Alpaca Eval 2.0 is an enhanced framework for evaluating instruction-following language models, '
-    'featuring an improved auto-annotator, updated baselines, and continuous preference calculation to '
-    'provide more accurate and cost-effective model assessments. '
-    'Currently not support `length-controlled winrate`; the official Judge model is `gpt-4-1106-preview`, while the baseline model is `gpt-4-turbo`.',  # noqa: E501
-    dataset_id='AI-ModelScope/alpaca_eval',
-    subset_list=['alpaca_eval_gpt4_baseline'],
-    metric_list=['winrate'],
-    few_shot_num=0,
-    train_split=None,
-    eval_split='eval')
-class AlpacaEvalAdapter(DataAdapter):
+@register_benchmark(
+    BenchmarkMeta(
+        name='alpaca_eval',
+        pretty_name='AlpacaEval2.0',
+        tags=[Tags.INSTRUCTION_FOLLOWING, Tags.ARENA],
+        description='Alpaca Eval 2.0 is an enhanced framework for evaluating instruction-following language models, '
+        'featuring an improved auto-annotator, updated baselines, and continuous preference calculation to '
+        'provide more accurate and cost-effective model assessments. '
+        'Currently not support `length-controlled winrate`; the official Judge model is `gpt-4-1106-preview`, while the baseline model is `gpt-4-turbo`.',  # noqa: E501
+        dataset_id='AI-ModelScope/alpaca_eval',
+        subset_list=['alpaca_eval_gpt4_baseline'],
+        metric_list=['winrate'],
+        few_shot_num=0,
+        train_split=None,
+        eval_split='eval',
+        prompt_template='{question}'
+    )
+)
+class AlpacaEvalAdapter(DefaultDataAdapter):
 
     def __init__(self, *args, **kwargs):
         super().__init__(*args, **kwargs)
 
-        # register metrics
-        metric_registry.register(Metric(name='winrate', object=mean))
-
-        # whether to use LLM as a judge
-        self.llm_as_a_judge = True
-
-    def gen_prompt(self, input_d: dict, subset_name: str, few_shot_list: list, **kwargs) -> dict:
-        question = input_d['instruction']
-        return self.gen_prompt_data(question)
-
-    def get_gold_answer(self, input_d: dict) -> str:
-        return input_d['output']
+        self._use_llm_judge = True  # Use LLM as a judge by default
+
+    def record_to_sample(self, record: Dict[str, Any]) -> Sample:
+        """
+        Convert a data record to a Sample object.
+
+        Args:
+            record (Dict[str, Any]): Input data record.
+
+        Returns:
+            Sample: Sample object with input, target, and metadata.
+        """
+        instruction = record['instruction']
+        baseline_output = record['output']  # baseline model output
+
+        return Sample(
+            input=instruction,
+            target=baseline_output,
+            metadata={
+                'generator': record.get('generator', 'unknown'),
+                'dataset': record.get('dataset', 'unknown')
+            }
+        )
+
+    def llm_match_score(
+        self,
+        original_prediction: str,
+        filtered_prediction: str,
+        reference: str,
+        task_state: TaskState,
+    ) -> Score:
+        score = Score(
+            extracted_prediction=filtered_prediction,
+            prediction=original_prediction,
+        )
+
+        instruction = task_state.input_text
+
+        # Request judge and obtain score
+        # reference is baseline answer 'm', filtered_prediction is model answer 'M'
+        prompt = GRADER_TEMPLATE.format(instruction=instruction, output_1=reference, output_2=filtered_prediction)
+        judge_response = self.llm_judge.judge(prompt, system_prompt=GRADER_SYSTEM_PROMPT)
 
-    def parse_pred_result(self, result: str, raw_input_d: dict = None, **kwargs) -> str:
-        return result.strip()
-
-    def match(self, gold: str, pred: str):
-        # simple match
-        logger.warning(f'Please use LLMJudge to match the result for {self.name}')
-        return None
-
-    def llm_match(self, gold: Any, pred: Any, judge: LLMJudge, **kwargs) -> bool:
-        raw_input = kwargs.get('raw_input', None)
-        instruction = raw_input['instruction']
-        # gold is baseline answer 'm', pred is model answer 'M'
-        prompt = GRADER_TEMPLATE.format(instruction=instruction, output_1=gold, output_2=pred)
-        # get grading response
-        grading_response = judge(prompt, system_prompt=GRADER_SYSTEM_PROMPT)
         # parse grading response
-        match = re.search(r'(m|M)', grading_response)
+        match = re.search(r'(m|M)', judge_response)
         res = match.group(0) if match else None
+
         if res:
-            return res == 'M'
+            winrate = 1 if res == 'M' else 0
         else:
-            logger.info(f'Failed to parse grading response: {prompt=}\n {grading_response=}')
-            return None
-
-    def compute_metric(self, review_res_list: List[bool], **kwargs) -> List[dict]:
-        # zip dict answers
-        res_list = [res for res in review_res_list if res is not None]
-
-        return super().compute_metric(res_list, **kwargs)
+            logger.info(f'Failed to parse grading response: {prompt=}\n {judge_response=}')
+            winrate = 0
+
+        # Set score based on the match result
+        score.value = {'winrate': winrate}
+        score.explanation = f'LLM judge: {judge_response}'
+        score.metadata = {
+            'source': 'llm_judge',
+            'judge_strategy': self.judge_strategy,
+            'model': self.llm_judge.model_id
+        }
+        score.main_score_name = 'winrate'
+        return score