evalscope 0.5.3__py3-none-any.whl → 0.5.5__py3-none-any.whl

evalscope/backend/rag_eval/cmteb/tasks/__init__.py

@@ -0,0 +1,64 @@
+from .Classification import *
+from .Clustering import *
+from .PairClassification import *
+from .Reranking import *
+from .Retrieval import *
+from .STS import *
+
+CLS_CLASSIFICATION = {
+    "TNews": TNews,
+    "IFlyTek": IFlyTek,
+    "MultilingualSentiment": MultilingualSentiment,
+    "JDReview": JDReview,
+    "OnlineShopping": OnlineShopping,
+    "Waimai": Waimai,
+}
+
+CLS_CLUSTERING = {
+    "CLSClusteringS2S": CLSClusteringFastS2S,
+    "CLSClusteringP2P": CLSClusteringFastP2P,
+    "ThuNewsClusteringS2S": ThuNewsClusteringFastS2S,
+    "ThuNewsClusteringP2P": ThuNewsClusteringFastP2P,
+}
+
+CLS_PAIR_CLASSIFICATION = {
+    "Ocnli": Ocnli,
+    "Cmnli": Cmnli,
+}
+
+CLS_RERANKING = {
+    "T2Reranking": T2Reranking,
+    "MMarcoReranking": MMarcoReranking,
+    "CMedQAv1": CMedQAv1,
+    "CMedQAv2": CMedQAv2,
+}
+
+CLS_RETRIEVAL = {
+    "T2Retrieval": T2Retrieval,
+    "MMarcoRetrieval": MMarcoRetrieval,
+    "DuRetrieval": DuRetrieval,
+    "CovidRetrieval": CovidRetrieval,
+    "CmedqaRetrieval": CmedqaRetrieval,
+    "EcomRetrieval": EcomRetrieval,
+    "MedicalRetrieval": MedicalRetrieval,
+    "VideoRetrieval": VideoRetrieval,
+}
+
+CLS_STS = {
+    "ATEC": ATEC,
+    "BQ": BQ,
+    "LCQMC": LCQMC,
+    "PAWSX": PAWSX,
+    "STSB": STSB,
+    "AFQMC": AFQMC,
+    "QBQTC": QBQTC,
+}
+
+CLS_DICT = {
+    **CLS_CLASSIFICATION,
+    **CLS_CLUSTERING,
+    **CLS_PAIR_CLASSIFICATION,
+    **CLS_RERANKING,
+    **CLS_RETRIEVAL,
+    **CLS_STS,
+}

evalscope/backend/rag_eval/ragas/__init__.py

@@ -0,0 +1,2 @@
+from evalscope.backend.rag_eval.ragas.arguments import TestsetGenerationArguments, EvaluationArguments
+from evalscope.backend.rag_eval.ragas.task_template import testset_generation, rag_eval

evalscope/backend/rag_eval/ragas/arguments.py

@@ -0,0 +1,37 @@
+from dataclasses import dataclass, field
+from typing import List, Optional, Union, Dict, Any
+
+
+@dataclass
+class TestsetGenerationArguments:
+    docs: List[str] = field(default_factory=list)
+    test_size: int = 10
+    output_file: str = "outputs/testset.json"
+    """
+    For local LLM support, you can use the following fields:
+        model_name_or_path: str
+        model_revision: str = "master"
+        template_type: str = "default"
+        generation_config: Optional[Dict]
+        
+    For API LLM support, you can use the following fields:
+        model_name="gpt-4o-mini"
+        api_base: str = "",
+        api_key: Optional[str] = None
+    """
+    generator_llm: Dict = field(default_factory=dict)
+    critic_llm: Dict = field(default_factory=dict)
+    embeddings: Dict = field(default_factory=dict)
+    distribution: str = field(
+        default_factory=lambda: {"simple": 0.5, "multi_context": 0.4, "reasoning": 0.1}
+    )
+
+
+@dataclass
+class EvaluationArguments:
+    testset_file: str
+    critic_llm: Dict = field(default_factory=dict)
+    embeddings: Dict = field(default_factory=dict)
+    metrics: List[str] = field(
+        default_factory=lambda: ["answer_relevancy", "faithfulness"]
+    )

evalscope/backend/rag_eval/ragas/task_template.py

@@ -0,0 +1,117 @@
+import os
+from evalscope.backend.rag_eval import EmbeddingModel, LLM
+from evalscope.utils.logger import get_logger
+from .arguments import TestsetGenerationArguments, EvaluationArguments
+
+logger = get_logger()
+
+
+def testset_generation(args: TestsetGenerationArguments) -> None:
+    from langchain_community.document_loaders.unstructured import UnstructuredFileLoader
+    from ragas.testset.generator import TestsetGenerator
+    from ragas.testset.evolutions import simple, reasoning, multi_context
+    from ragas import RunConfig
+
+    # load data
+    file_path = args.docs
+    loader = UnstructuredFileLoader(file_path, mode="elements")
+    data = loader.load()
+
+    # generator with models
+    generator_llm = LLM.load(**args.generator_llm)
+    critic_llm = LLM.load(**args.critic_llm)
+    embeddings = EmbeddingModel.load(**args.embeddings)
+
+    # Change resulting question type distribution
+    distributions = {
+        simple: args.distribution["simple"],
+        multi_context: args.distribution["multi_context"],
+        reasoning: args.distribution["reasoning"],
+    }
+
+    generator = TestsetGenerator.from_langchain(generator_llm, critic_llm, embeddings)
+
+    runconfig = RunConfig(timeout=30, max_retries=1, max_wait=30, max_workers=1)
+    testset = generator.generate_with_langchain_docs(
+        data,
+        args.test_size,
+        distributions,
+        with_debugging_logs=True,
+        is_async=False,
+        run_config=runconfig,
+    )
+
+    # save file
+    testset_df = testset.to_pandas()
+    output_path = os.path.dirname(args.output_file)
+    os.makedirs(output_path, exist_ok=True)
+    testset_df.to_json(args.output_file, indent=4, index=False, orient="records")
+
+    # get answer
+    testset_with_answer = get_answer(testset_df, generator_llm)
+    testset_with_answer.to_json(
+        args.output_file, indent=4, index=False, orient="records"
+    )
+
+
+def get_answer(testset_df, generator_llm):
+    template = """You are an assistant for question-answering tasks. 
+Use the following pieces of retrieved context to answer the question. 
+If you don't know the answer, just say that you don't know. 
+Use two sentences maximum and keep the answer concise.
+Question: {question} 
+Context: {contexts} 
+Answer:
+"""
+    answers = []
+    for index, row in testset_df.iterrows():
+        question = row["question"]
+        contexts = "\n".join(row["contexts"])
+
+        # Combine question and contexts as input for the LLM
+        input_text = template.format(question=question, contexts=contexts)
+
+        # Generate the answer using the generator LLM
+        answer = generator_llm.invoke(input_text)
+        answers.append(answer)
+
+    testset_df["answer"] = answers
+    return testset_df
+
+
+def rag_eval(
+    args: EvaluationArguments,
+) -> None:
+    from datasets import Dataset
+    from ragas import evaluate
+    from evalscope.backend.rag_eval import EmbeddingModel, LLM
+    from ragas import RunConfig
+    import importlib
+
+    def dynamic_import(module_name, *function_names):
+        # 动态导入指定模块
+        module = importlib.import_module(module_name)
+
+        functions = [getattr(module, name) for name in function_names]
+        return functions
+
+    llm = LLM.load(**args.critic_llm)
+    embedding = EmbeddingModel.load(**args.embeddings)
+
+    dataset = Dataset.from_json(args.testset_file)
+
+    runconfig = RunConfig(timeout=30, max_retries=1, max_wait=30, max_workers=1)
+    score = evaluate(
+        dataset,
+        metrics=dynamic_import("ragas.metrics", *args.metrics),
+        llm=llm,
+        embeddings=embedding,
+        run_config=runconfig,
+    )
+    score_df = score.to_pandas()
+    # logger.info(score_df.to_string())
+
+    output_path = args.testset_file.split(".")[0] + "_score.json"
+    score_df.to_json(output_path, indent=4, index=False, orient="records")
+
+    logger.info(f"Eval score saved to {output_path}")

evalscope/backend/vlm_eval_kit/backend_manager.py

@@ -1,10 +1,9 @@
 from typing import Optional, Union
-from evalscope.utils import is_module_installed, get_module_path, get_valid_list, yaml_to_dict, json_to_dict
+from evalscope.utils import is_module_installed, get_valid_list
 from evalscope.backend.base import BackendManager
 from evalscope.utils.logger import get_logger
 from functools import partial
 import subprocess
-from dataclasses import dataclass
 import copy
 
 logger = get_logger()

evalscope/backend/vlm_eval_kit/custom_dataset.py

@@ -8,7 +8,7 @@ class CustomDataset:
 
     def load_data(self, dataset):
         # customize the loading of the dataset
-        data_path = os.path.join("~/LMUData", f'{dataset}.tsv')
+        data_path = os.path.join(os.path.expanduser("~/LMUData"), f'{dataset}.tsv')
         return load(data_path)
 
 

evalscope/benchmarks/benchmark.py

@@ -46,7 +46,7 @@ class Benchmark(object):
 
             dataset.dataset_name = dataset_name.split('/')[-1]
             dataset.subset_name = subset
-            dataset.split = split
+            # dataset.split = split
             return dataset
         elif hub == 'HuggingFace':
             # TODO: implement this by xingjun.wxj@alibaba-inc.com

evalscope/evaluator/evaluator.py

@@ -174,6 +174,7 @@ class Evaluator(object):
         """
         assert self.data_adapter is not None, 'data_adapter must be provided when calling func get_answers() !'
         assert self.model_adapter is not None, 'model must be provided when calling func get_answers() !'
+        assert len(prompts_list) > 0, 'prompts_list must not be empty when calling func get_answers() !'
 
         answers_list = []
         pred_dir: str = self.outputs_structure.get(OutputsStructure.PREDICTIONS_DIR)
@@ -244,8 +245,8 @@ class Evaluator(object):
                 answer_d[AnswerKeys.ORIGIN_PROMPT] = input_prompt
 
                 if debug:
-                    logger.debug(f'**input_prompt: {json.dumps(input_prompt, ensure_ascii=False)} \n')
-                    logger.debug(f'**predicted ans: {json.dumps(answer_d, ensure_ascii=False)} \n')
+                    logger.info(f'**input_prompt: {json.dumps(input_prompt, ensure_ascii=False)} \n')
+                    logger.info(f'**predicted ans: {json.dumps(answer_d, ensure_ascii=False)} \n')
 
                 answers_list.append(answer_d)
 
@@ -349,7 +350,7 @@ class Evaluator(object):
             review_d = self._get_review(answer_d=answer_d, review_id=review_id, reviewer_spec=reviewer_spec)
 
             if debug:
-                logger.debug(review_d)
+                logger.info(review_d)
 
             reviews_list.append(review_d)
 

evalscope/metrics/bundled_rouge_score/rouge_scorer.py

@@ -31,6 +31,7 @@ In these examples settings.xml lists input files and formats.
 from __future__ import absolute_import, division, print_function
 import collections
 import re
+import os
 
 import nltk
 import numpy as np
@@ -38,6 +39,24 @@ import six
 from absl import logging
 from rouge_score import scoring, tokenizers
 from six.moves import map, range
+from evalscope.utils import get_logger
+
+logger = get_logger()
+
+# Deal with nltk punkt_tab.zip tokenizer file to avoid downloading issue
+try:
+    nltk_dir = os.path.join(os.path.expanduser('~'), 'nltk_data/tokenizers')
+    os.makedirs(nltk_dir, exist_ok=True)
+    punkt_path = os.path.join(nltk_dir, 'punkt_tab.zip')
+    punkt_tab_url = 'https://modelscope-open.oss-cn-hangzhou.aliyuncs.com/open_data/nltk_data/punkt_tab.zip'
+
+    if not os.path.exists(punkt_path):
+        os.system(f'wget -P {nltk_dir} {punkt_tab_url}')
+        os.system(f'unzip {punkt_path} -d {nltk_dir}')
+    else:
+        logger.info(f'{punkt_path} already exists, skipping download')
+except Exception as e:
+    logger.error(f'Try to download punkt_tab.zip for nltk failed: {e}')
 
 
 class RougeScorer(scoring.BaseScorer):

evalscope/models/api/__init__.py

@@ -0,0 +1,3 @@
+# Copyright (c) Alibaba, Inc. and its affiliates.
+
+from evalscope.models.api.openai_api import OpenaiApi

evalscope/models/api/openai_api.py

@@ -0,0 +1,228 @@
+# Copyright (c) Alibaba, Inc. and its affiliates.
+
+import json
+import threading
+import time
+from asyncio import Queue
+
+import requests
+from typing import Union, List, Optional, Dict
+from concurrent.futures import ThreadPoolExecutor
+from modelscope.utils.logger import get_logger
+
+logger = get_logger()
+
+
+class OpenaiApi:
+
+    def __init__(self,
+                 model: str,
+                 openai_api_key,
+                 openai_api_base,
+                 logprobs: Optional[bool] = False,
+                 top_logprobs: Optional[int] = None,
+                 max_new_tokens: int = 4096,
+                 temperature: Optional[float] = 0.0,
+                 repetition_penalty: Optional[float] = 1.0,
+                 is_chat: bool = True,
+                 verbose: bool = True,
+                 retry: int = 3,
+                 query_per_second: int = 10,     # TODO
+                 **kwargs):
+
+        self.temperature = temperature
+        self.repetition_penalty = repetition_penalty
+        self.max_tokens = max_new_tokens
+        self.logprobs = logprobs
+        self.top_logprobs = top_logprobs
+
+        self.openai_api_key = openai_api_key
+        self.url = openai_api_base
+        self.model = model
+        self.is_chat = is_chat
+        self.retry = retry
+        self.verbose = verbose
+
+        self.token_bucket = TokenBucket(query_per_second, verbose)
+
+    def generate_simple(self, inputs: Union[List[str]]):
+
+        def process_one(in_data: str):
+
+            if self.is_chat:
+                data = dict(
+                    model=self.model,
+                    messages=[{'role': 'user', 'content': in_data}],
+                    max_tokens=self.max_tokens,
+                    n=1,
+                    logprobs=self.logprobs,
+                    top_logprobs=self.top_logprobs,
+                    stop=None,
+                    temperature=self.temperature,
+                    repetition_penalty=self.repetition_penalty,
+                )
+            else:
+                data = dict(
+                    model=self.model,
+                    prompt=in_data,
+                    max_tokens=self.max_tokens,
+                    temperature=self.temperature,
+                    repetition_penalty=self.repetition_penalty,
+                )
+
+            # todo
+            openai_api_key = self.openai_api_key or ''
+            header = {'Authorization': f'Bearer ', 'content-type': 'application/json', }
+            data = json.dumps(data, ensure_ascii=False)
+
+            if self.verbose:
+                print(f'>>data in generate_simple: {data}')
+
+            resp = requests.post(self.url, headers=header, data=data)
+            resp = resp.json()
+            if self.verbose:
+                print(f'>>resp in generate_simple: {resp}')
+
+            if self.logprobs:
+                return resp['choices']
+            else:
+                if self.is_chat:
+                    return resp['choices'][0]['message']['content'].strip()
+                else:
+                    return resp['choices'][0]['text'].strip()
+
+        with ThreadPoolExecutor() as executor:
+            results = list(executor.map(process_one, inputs))
+
+        return results
+
+    def generate(self,
+                 inputs: Union[List[str], List[List]],
+                 **kwargs) -> List[str]:
+        """
+        Generate responses from OpenAI API.
+
+        Args:
+            inputs: The input messages for the model. It can be a string or a list of messages.
+                e.g. ['who are you ?', 'what is your name ?']
+                e.g. [[{'role': 'user', 'content': 'who are you ?'}], ...]
+            kwargs: The optional arguments for the model.
+        """
+        results = []
+        # with ThreadPoolExecutor() as executor:
+        #     results = list(executor.map(self._generate, inputs))
+
+        for input in inputs:
+            results.append(self._generate(input))
+
+        return results
+
+    def _generate(self, messages: Union[str, List[Dict]]) -> str:
+
+        if isinstance(messages, str):
+            messages = [{'role': 'user', 'content': messages}]
+
+        max_num_retries = 0
+        while max_num_retries < self.retry:
+            # self.wait()
+
+            header = {
+                'Authorization': f'Bearer {self.openai_api_key}',
+                'content-type': 'application/json',
+            }
+
+            try:
+                if self.is_chat:
+                    data = dict(
+                        model=self.model,
+                        messages=messages,
+                        max_tokens=self.max_tokens,
+                        n=1,
+                        logprobs=self.logprobs,
+                        top_logprobs=self.top_logprobs,
+                        stop=None,
+                        temperature=self.temperature,
+                        repetition_penalty=self.repetition_penalty,
+                    )
+                else:
+                    # TODO: This is a temporary solution for non-chat models.
+                    input_prompts = []
+                    for msg in messages:
+                        input_prompts.append(msg['content'])
+
+                    data = dict(
+                        model=self.model,
+                        prompt='\n'.join(input_prompts),
+                        max_tokens=self.max_tokens,
+                        temperature=self.temperature,
+                        repetition_penalty=self.repetition_penalty,
+                    )
+
+                def remove_none_val(input_d: dict):
+                    return {k: v for k, v in input_d.items() if v is not None}
+                data = remove_none_val(data)
+
+                if self.verbose:
+                    logger.info(f'>> Post data: {json.dumps(data, ensure_ascii=False)}')
+                raw_response = requests.post(self.url,
+                                             headers=header,
+                                             data=json.dumps(data, ensure_ascii=False))
+
+                response = raw_response.json()
+                if self.verbose:
+                    logger.info(f'>> response: {response}')
+
+                if self.logprobs:
+                    return response['choices']
+                else:
+                    if self.is_chat:
+                        return response['choices'][0]['message']['content'].strip()
+                    else:
+                        return response['choices'][0]['text'].strip()
+
+            except Exception as e:
+                logger.error(f'Error occurs: {str(e)}')
+                max_num_retries += 1
+                continue
+
+    def wait(self):
+        return self.token_bucket.get_token()
+
+
+class TokenBucket:
+    """A token bucket for rate limiting.
+
+    Args:
+        query_per_second (float): The rate of the token bucket.
+    """
+
+    def __init__(self, rate, verbose=False):
+        self._rate = rate
+        self._tokens = threading.Semaphore(0)
+        self.started = False
+        self._request_queue = Queue()
+        self.logger = get_logger()
+        self.verbose = verbose
+
+    def _add_tokens(self):
+        """Add tokens to the bucket."""
+        while True:
+            if self._tokens._value < self._rate:
+                self._tokens.release()
+            time.sleep(1 / self._rate)
+
+    def get_token(self):
+        """Get a token from the bucket."""
+        if not self.started:
+            self.started = True
+            threading.Thread(target=self._add_tokens, daemon=True).start()
+        self._tokens.acquire()
+        if self.verbose:
+            cur_time = time.time()
+            while not self._request_queue.empty():
+                if cur_time - self._request_queue.queue[0] > 60:
+                    self._request_queue.get()
+                else:
+                    break
+            self._request_queue.put(cur_time)
+            self.logger.info(f'Current RPM {self._request_queue.qsize()}.')

evalscope/perf/http_client.py

@@ -51,15 +51,15 @@ UNLIMITED_RATE = -1
 
 
 async def on_request_start(session, context, params):
-    logger.debug(f'Starting request: <{params}>')
+    logger.info(f'Starting request: <{params}>')
 
 
 async def on_request_chunk_sent(session, context, params):
-    logger.debug(f'Request body: {params}')
+    logger.info(f'Request body: {params}')
 
 
 async def on_response_chunk_received(session, context, params):
-    logger.debug(f'Response info: <{params}>')
+    logger.info(f'Response info: <{params}>')
 
 
 class AioHttpClient:
@@ -116,7 +116,7 @@ class AioHttpClient:
                 line = line.decode("utf8")
                 line = line.rstrip("\n").rstrip("\r")
                 if self.debug:
-                    logger.debug(line)
+                    logger.info(line)
                 sse_msg = ServerSentEvent.decode(line)
                 if not sse_msg:
                     continue
@@ -567,7 +567,7 @@ async def send_requests_worker(task_id, request_queue: asyncio.Queue, benchmark_
                     else:
                         if response_data:
                             collected_messages.append(response_data)  # save the message
-                            logger.debug(response_data)
+                            logger.info(response_data)
                             benchmark_data["chunk_times"].append(time.perf_counter())
 
                 benchmark_data["response_messages"] = collected_messages

evalscope/run.py

@@ -207,6 +207,10 @@ def run_task(task_cfg: Union[str, dict, TaskConfig, List[TaskConfig]]) -> Union[
             from evalscope.backend.vlm_eval_kit import VLMEvalKitBackendManager
             vlm_eval_kit_backend_manager = VLMEvalKitBackendManager(config=eval_config)
             vlm_eval_kit_backend_manager.run()
+        elif eval_backend == EvalBackend.RAG_EVAL.value:
+            from evalscope.backend.rag_eval import RAGEvalBackendManager
+            rag_eval_backend_manager = RAGEvalBackendManager(config=eval_config)
+            rag_eval_backend_manager.run()
         # TODO: Add other evaluation backends
         elif eval_backend == EvalBackend.THIRD_PARTY.value:
             raise NotImplementedError(f'Not implemented for evaluation backend {eval_backend}')

evalscope/third_party/longbench_write/__init__.py

@@ -0,0 +1,3 @@
+# Copyright (c) Alibaba, Inc. and its affiliates.
+
+from evalscope.third_party.longbench_write.longbench_write import run_task