evalscope 0.5.5rc1__py3-none-any.whl → 0.6.0__py3-none-any.whl

evalscope/backend/rag_eval/ragas/tasks/testset_generation.py

@@ -0,0 +1,263 @@
+import os
+import asyncio
+import pandas as pd
+from tqdm import tqdm
+from ragas.llms import LangchainLLMWrapper
+from ragas.embeddings import LangchainEmbeddingsWrapper
+from .translate_prompt import translate_prompts
+from evalscope.utils.logger import get_logger
+from evalscope.backend.rag_eval.ragas.arguments import TestsetGenerationArguments
+from evalscope.backend.rag_eval import EmbeddingModel, LLM, ChatOpenAI
+
+os.environ['DO_NOT_TRACK'] = 'true'
+
+logger = get_logger()
+
+
+def get_transform(llm, embedding, language):
+    """
+    Creates and returns a default set of transforms for processing a knowledge graph.
+
+    This function defines a series of transformation steps to be applied to a
+    knowledge graph, including extracting summaries, keyphrases, titles,
+    headlines, and embeddings, as well as building similarity relationships
+    between nodes.
+
+    The transforms are applied in the following order:
+    1. Parallel extraction of summaries and headlines
+    2. Embedding of summaries for document nodes
+    3. Splitting of headlines
+    4. Parallel extraction of embeddings, keyphrases, and titles
+    5. Building cosine similarity relationships between nodes
+    6. Building cosine similarity relationships between summaries
+
+    Returns
+    -------
+    Transforms
+        A list of transformation steps to be applied to the knowledge graph.
+
+    """
+    from ragas.testset.transforms.engine import Parallel
+    from ragas.testset.transforms.extractors import (
+        EmbeddingExtractor,
+        HeadlinesExtractor,
+        KeyphrasesExtractor,
+        SummaryExtractor,
+        TitleExtractor,
+    )
+    from ragas.testset.transforms.relationship_builders.cosine import (
+        CosineSimilarityBuilder,
+        SummaryCosineSimilarityBuilder,
+    )
+    from ragas.testset.transforms.splitters import HeadlineSplitter
+    from ragas.testset.graph import NodeType
+
+    # define the transforms
+    summary_extractor = SummaryExtractor(llm=llm)
+    keyphrase_extractor = KeyphrasesExtractor(llm=llm)
+    title_extractor = TitleExtractor(llm=llm)
+    headline_extractor = HeadlinesExtractor(llm=llm)
+
+    asyncio.run(
+        translate_prompts(
+            prompts=[
+                summary_extractor,
+                keyphrase_extractor,
+                title_extractor,
+                headline_extractor,
+            ],
+            target_lang=language,
+            llm=llm,
+            adapt_instruction=True,
+        )
+    )
+
+    embedding_extractor = EmbeddingExtractor(embedding_model=embedding)
+    headline_splitter = HeadlineSplitter()
+    cosine_sim_builder = CosineSimilarityBuilder(threshold=0.8)
+    summary_embedder = EmbeddingExtractor(
+        name='summary_embedder',
+        filter_nodes=lambda node: True if node.type == NodeType.DOCUMENT else False,
+        property_name='summary_embedding',
+        embed_property_name='summary',
+        embedding_model=embedding,
+    )
+    summary_cosine_sim_builder = SummaryCosineSimilarityBuilder(threshold=0.6)
+
+    # specify the transforms and their order to be applied
+    transforms = [
+        Parallel(summary_extractor, headline_extractor),
+        summary_embedder,
+        headline_splitter,
+        Parallel(embedding_extractor, keyphrase_extractor, title_extractor),
+        cosine_sim_builder,
+        summary_cosine_sim_builder,
+    ]
+    return transforms
+
+
+def get_distribution(llm, distribution, language):
+    from ragas.testset.synthesizers.abstract_query import (
+        AbstractQuerySynthesizer,
+        ComparativeAbstractQuerySynthesizer,
+    )
+    from ragas.testset.synthesizers.specific_query import SpecificQuerySynthesizer
+
+    abstract = AbstractQuerySynthesizer(llm=llm)
+    comparative = ComparativeAbstractQuerySynthesizer(llm=llm)
+    specific = SpecificQuerySynthesizer(llm=llm)
+
+    asyncio.run(
+        translate_prompts(
+            prompts=[
+                abstract,
+                comparative,
+                specific,
+            ],
+            target_lang=language,
+            llm=llm,
+            adapt_instruction=True,
+        )
+    )
+    return [
+        (abstract, distribution['simple']),
+        (comparative, distribution['multi_context']),
+        (specific, distribution['reasoning']),
+    ]
+
+
+def get_knowledge_graph(documents, transforms, local_file):
+    from ragas.testset.graph import KnowledgeGraph, Node, NodeType
+    from ragas.testset.transforms import apply_transforms
+
+    if os.path.exists(local_file):
+        logger.info(f'Loading knowledge graph from {local_file}')
+        return KnowledgeGraph.load(local_file)
+    # convert the documents to Ragas nodes
+    nodes = []
+    for doc in documents:
+        node = Node(
+            type=NodeType.DOCUMENT,
+            properties={
+                'page_content': doc.page_content,
+                'document_metadata': doc.metadata,
+            },
+        )
+        nodes.append(node)
+
+    kg = KnowledgeGraph(nodes=nodes)
+
+    # apply transforms and update the knowledge graph
+    apply_transforms(kg, transforms)
+
+    # save the knowledge graph
+    output_path = os.path.dirname(local_file)
+    os.makedirs(output_path, exist_ok=True)
+    kg.save(local_file)
+    logger.info(f'Knowledge graph saved to {local_file}')
+    return kg
+
+
+def load_data(file_path):
+    from langchain_community.document_loaders import UnstructuredFileLoader
+
+    loader = UnstructuredFileLoader(file_path, mode='elements')
+    data = loader.load()
+    return data
+
+
+def generate_testset(args: TestsetGenerationArguments) -> None:
+
+    from ragas.testset import TestsetGenerator
+    from ragas import RunConfig
+
+    # load data
+    documents = load_data(args.docs)
+
+    # generator with models
+    generator_llm = LLM.load(**args.generator_llm)
+    embeddings = EmbeddingModel.load(**args.embeddings)
+
+    # Change resulting question type distribution
+    distributions = get_distribution(
+        LangchainLLMWrapper(generator_llm), args.distribution, args.language
+    )
+
+    # get transforms
+    transforms = get_transform(
+        LangchainLLMWrapper(generator_llm),
+        LangchainEmbeddingsWrapper(embeddings),
+        args.language,
+    )
+
+    # get knowledge graph
+    knowledge_graph = get_knowledge_graph(documents, transforms, args.knowledge_graph)
+
+    generator = TestsetGenerator.from_langchain(
+        generator_llm, embeddings, knowledge_graph
+    )
+
+    runconfig = RunConfig(
+        timeout=600, max_retries=3, max_wait=120, max_workers=1, log_tenacity=True
+    )
+    testset = generator.generate(
+        testset_size=args.test_size,
+        query_distribution=distributions,
+        run_config=runconfig,
+        with_debugging_logs=True,
+        raise_exceptions=True,
+    )
+
+    # save file
+    testset_df = testset.to_pandas()
+    output_path = os.path.dirname(args.output_file)
+    os.makedirs(output_path, exist_ok=True)
+    testset_df.to_json(
+        args.output_file, indent=4, index=False, orient='records', force_ascii=False
+    )
+
+    # get answer
+    testset_with_answer = get_answer(testset_df, generator_llm, args.language)
+    testset_with_answer.to_json(
+        args.output_file.replace('.json', '_with_answer.json'),
+        indent=4,
+        index=False,
+        orient='records',
+        force_ascii=False,
+    )
+
+
+def get_answer(testset_df, generator_llm, language: None):
+    template = """You are an assistant for question-answering tasks.
+Use the following pieces of retrieved context to answer the question.
+If you don't know the answer, just say that you don't know. Answer in {language}.
+Question: {question}
+Context: {contexts}
+Answer:
+"""
+
+    items = []
+    for i in tqdm(range(len(testset_df)), desc='Generating Answers'):
+        row = testset_df.iloc[i]
+        question = row['user_input']
+        contexts = '\n'.join(row['reference_contexts'])
+
+        # Combine question and contexts as input for the LLM
+        input_text = template.format(
+            language=language, question=question, contexts=contexts
+        )
+
+        # Generate the answer using the generator LLM
+        answer = generator_llm.invoke(input_text)
+        if isinstance(generator_llm, ChatOpenAI):
+            answer = answer.content
+        items.append(
+            {
+                'user_input': question,
+                'retrieved_contexts': row['reference_contexts'],
+                'response': answer,
+                'reference': row['reference'],
+            }
+        )
+
+    return pd.DataFrame.from_dict(items)

evalscope/backend/rag_eval/ragas/tasks/translate_prompt.py

@@ -0,0 +1,72 @@
+import os
+import asyncio
+from typing import List
+from ragas.prompt import PromptMixin
+from ragas.llms import BaseRagasLLM
+from ragas.utils import RAGAS_SUPPORTED_LANGUAGE_CODES
+from evalscope.utils.logger import get_logger
+
+
+logger = get_logger()
+
+
+async def translate_prompt(
+    prompt_user: PromptMixin,
+    target_lang: str,
+    llm: BaseRagasLLM,
+    adapt_instruction: bool = False,
+):
+    if target_lang not in RAGAS_SUPPORTED_LANGUAGE_CODES:
+        logger.warning(
+            f'{target_lang} is not in supported language: {list(RAGAS_SUPPORTED_LANGUAGE_CODES)}'
+        )
+        return
+
+    if not issubclass(type(prompt_user), PromptMixin):
+        logger.info(f"{prompt_user} is not a PromptMixin, don't translate it")
+        return
+
+    class_name = prompt_user.__class__.__name__
+    current_dir = os.path.dirname(__file__)
+    prompt_dir = os.path.abspath(
+        os.path.join(current_dir, f'../prompts/{target_lang}/{class_name}')
+    )
+    os.makedirs(prompt_dir, exist_ok=True)
+
+    try:
+        loader_prompts = prompt_user.load_prompts(prompt_dir, target_lang)
+        prompt_user.set_prompts(**loader_prompts)
+        logger.info(f'Load existing prompts from {prompt_dir}')
+        return
+    except FileNotFoundError:
+        logger.info(f'Not find existing prompts {class_name}, generate new prompts.')
+
+    logger.info(f'Translating prompts to {target_lang}')
+    adapted_prompts = await prompt_user.adapt_prompts(
+        language=target_lang, llm=llm, adapt_instruction=adapt_instruction
+    )
+    prompt_user.set_prompts(**adapted_prompts)
+    try:
+        prompt_user.save_prompts(prompt_dir)
+    except FileExistsError:
+        logger.info(f'Find existing prompt {class_name}, skip saving.')
+    logger.info(f'Save new prompts to {prompt_dir}')
+
+    return
+
+
+async def translate_prompts(
+    prompts: List[PromptMixin],
+    target_lang: str,
+    llm: BaseRagasLLM,
+    adapt_instruction: bool = False,
+):
+    if target_lang and target_lang != 'english':
+        await asyncio.gather(
+            *[
+                translate_prompt(prompt, target_lang, llm, adapt_instruction)
+                for prompt in prompts
+            ]
+        )
+
+        logger.info('Translate prompts finished')

evalscope/backend/vlm_eval_kit/backend_manager.py

@@ -4,7 +4,6 @@ from evalscope.backend.base import BackendManager
 from evalscope.utils.logger import get_logger
 from functools import partial
 import subprocess
-from dataclasses import dataclass
 import copy
 
 logger = get_logger()

evalscope/backend/vlm_eval_kit/custom_dataset.py

@@ -8,7 +8,7 @@ class CustomDataset:
 
     def load_data(self, dataset):
         # customize the loading of the dataset
-        data_path = os.path.join("~/LMUData", f'{dataset}.tsv')
+        data_path = os.path.join(os.path.expanduser("~/LMUData"), f'{dataset}.tsv')
         return load(data_path)
 
 

evalscope/evaluator/evaluator.py

@@ -174,6 +174,7 @@ class Evaluator(object):
         """
         assert self.data_adapter is not None, 'data_adapter must be provided when calling func get_answers() !'
         assert self.model_adapter is not None, 'model must be provided when calling func get_answers() !'
+        assert len(prompts_list) > 0, 'prompts_list must not be empty when calling func get_answers() !'
 
         answers_list = []
         pred_dir: str = self.outputs_structure.get(OutputsStructure.PREDICTIONS_DIR)

evalscope/models/api/openai_api.py

@@ -76,12 +76,12 @@ class OpenaiApi:
             data = json.dumps(data, ensure_ascii=False)
 
             if self.verbose:
-                print(f'>>data in generate_simple: {data}')
+                logger.info(f'>>data in generate_simple: {data}')
 
             resp = requests.post(self.url, headers=header, data=data)
             resp = resp.json()
             if self.verbose:
-                print(f'>>resp in generate_simple: {resp}')
+                logger.info(f'>>resp in generate_simple: {resp}')
 
             if self.logprobs:
                 return resp['choices']

evalscope/perf/http_client.py

@@ -209,7 +209,7 @@ async def dispatch_requests_worker(request_queue: asyncio.Queue, args):
                 prompt = f.read()
         else:
             prompt = args.prompt
-        messages = {'role': 'user', 'content': prompt}
+        messages = [{'role': 'user', 'content': prompt}]
         request = query_generator.build_request(messages, query_parameters)
         if args.number is None:
             await request_queue.put(request)

evalscope/perf/openai_api.py

@@ -39,6 +39,8 @@ class OpenaiPlugin(ApiPluginBase):
         try:
             if param.query_template is not None:
                 query = json.loads(param.query_template)
+                if 'stream' in query.keys():
+                    param.stream = query['stream']
                 query['messages'] = messages   # replace template messages with input messages.
                 return self.__compose_query_from_parameter(query, param)
             else:

evalscope/run.py

@@ -207,6 +207,10 @@ def run_task(task_cfg: Union[str, dict, TaskConfig, List[TaskConfig]]) -> Union[
             from evalscope.backend.vlm_eval_kit import VLMEvalKitBackendManager
             vlm_eval_kit_backend_manager = VLMEvalKitBackendManager(config=eval_config)
             vlm_eval_kit_backend_manager.run()
+        elif eval_backend == EvalBackend.RAG_EVAL.value:
+            from evalscope.backend.rag_eval import RAGEvalBackendManager
+            rag_eval_backend_manager = RAGEvalBackendManager(config=eval_config)
+            rag_eval_backend_manager.run()
         # TODO: Add other evaluation backends
         elif eval_backend == EvalBackend.THIRD_PARTY.value:
             raise NotImplementedError(f'Not implemented for evaluation backend {eval_backend}')

evalscope/utils/logger.py

@@ -1,18 +1,20 @@
 # Copyright (c) Alibaba, Inc. and its affiliates.
 
+import importlib.util as iutil
 import logging
 from typing import Optional
 
 init_loggers = {}
+format = "%(asctime)s - %(name)s - %(levelname)s - %(message)s"
+formatter = logging.Formatter(format)
 
-formatter = logging.Formatter(
-    '%(asctime)s - %(name)s - %(levelname)s - %(message)s')
+logging.basicConfig(format=format, level=logging.INFO)
 
 
-def get_logger(log_file: Optional[str] = None,
-               log_level: int = logging.INFO,
-               file_mode: str = 'w'):
-    """ Get logging logger
+def get_logger(
+    log_file: Optional[str] = None, log_level: int = logging.INFO, file_mode: str = "w"
+):
+    """Get logging logger
 
     Args:
         log_file: Log filename, if specified, file handler will be added to
@@ -22,21 +24,39 @@ def get_logger(log_file: Optional[str] = None,
             specified (if filemode is unspecified, it defaults to 'w').
     """
 
-    logger_name = __name__.split('.')[0]
+    logger_name = __name__.split(".")[0]
     logger = logging.getLogger(logger_name)
-
+    logger.propagate = False
     if logger_name in init_loggers:
         add_file_handler_if_needed(logger, log_file, file_mode, log_level)
+        if logger.level != log_level:
+            logger.setLevel(log_level)
         return logger
 
-    for handler in logger.root.handlers:
-        if type(handler) is logging.StreamHandler:
-            handler.setLevel(logging.ERROR)
+    # handle duplicate logs to the console
+    # Starting in 1.8.0, PyTorch DDP attaches a StreamHandler <stderr> (NOTSET)
+    # to the root logger. As logger.propagate is True by default, this root
+    # level handler causes logging messages from rank>0 processes to
+    # unexpectedly show up on the console, creating much unwanted clutter.
+    # To fix this issue, we set the root logger's StreamHandler, if any, to log
+    # at the ERROR level.
+    torch_dist = False
+    is_worker0 = True
+    if iutil.find_spec("torch") is not None:
+        from modelscope.utils.torch_utils import is_dist, is_master
+
+        torch_dist = is_dist()
+        is_worker0 = is_master()
+
+    if torch_dist:
+        for handler in logger.root.handlers:
+            if type(handler) is logging.StreamHandler:
+                handler.setLevel(logging.ERROR)
 
     stream_handler = logging.StreamHandler()
     handlers = [stream_handler]
 
-    if log_file is not None:
+    if is_worker0 and log_file is not None:
         file_handler = logging.FileHandler(log_file, file_mode)
         handlers.append(file_handler)
 
@@ -45,7 +65,10 @@ def get_logger(log_file: Optional[str] = None,
         handler.setLevel(log_level)
         logger.addHandler(handler)
 
-    logger.setLevel(log_level)
+    if is_worker0:
+        logger.setLevel(log_level)
+    else:
+        logger.setLevel(logging.ERROR)
 
     init_loggers[logger_name] = True
 
@@ -57,7 +80,14 @@ def add_file_handler_if_needed(logger, log_file, file_mode, log_level):
         if isinstance(handler, logging.FileHandler):
             return
 
-    if log_file is not None:
+    if iutil.find_spec("torch") is not None:
+        from modelscope.utils.torch_utils import is_master
+
+        is_worker0 = is_master()
+    else:
+        is_worker0 = True
+
+    if is_worker0 and log_file is not None:
         file_handler = logging.FileHandler(log_file, file_mode)
         file_handler.setFormatter(formatter)
         file_handler.setLevel(log_level)

evalscope/utils/task_utils.py

@@ -11,6 +11,9 @@ class EvalBackend(Enum):
     
     # Use VLM Eval Kit as the multi-modal model evaluation backend
     VLM_EVAL_KIT = 'VLMEvalKit'
+    
+    # Use RAGEval as the RAG evaluation backend
+    RAG_EVAL = 'RAGEval'
 
     # Use third-party evaluation backend/modules
     THIRD_PARTY = 'ThirdParty'

evalscope/version.py

@@ -1,4 +1,4 @@
 # Copyright (c) Alibaba, Inc. and its affiliates.
 
-__version__ = '0.5.5rc1'
-__release_datetime__ = '2024-09-29 08:00:00'
+__version__ = "0.6.0"
+__release_datetime__ = "2024-11-08 11:59:59"