evalscope 0.6.0rc0__py3-none-any.whl → 0.7.0__py3-none-any.whl

evalscope/backend/rag_eval/ragas/prompts/chinese/MultiHopAbstractQuerySynthesizer/theme_persona_matching_prompt_chinese.json

@@ -0,0 +1,39 @@
+{
+  "ragas_version": "0.2.5",
+  "original_hash": 2334929353739018813,
+  "language": "chinese",
+  "instruction": "给定一个主题和角色列表，根据角色描述将每个角色与相关主题关联起来。",
+  "examples": [
+    {
+      "input": {
+        "themes": [
+          "同理心",
+          "包容性",
+          "远程工作"
+        ],
+        "personas": [
+          {
+            "name": "人力资源经理",
+            "role_description": "专注于包容性和员工支持。"
+          },
+          {
+            "name": "远程团队负责人",
+            "role_description": "管理远程团队沟通。"
+          }
+        ]
+      },
+      "output": {
+        "mapping": {
+          "HR Manager": [
+            "包容性",
+            "同理心"
+          ],
+          "Remote Team Lead": [
+            "远程工作",
+            "同理心"
+          ]
+        }
+      }
+    }
+  ]
+}

evalscope/backend/rag_eval/ragas/prompts/chinese/MultiHopSpecificQuerySynthesizer/generate_query_reference_prompt_chinese.json

@@ -0,0 +1,7 @@
+{
+  "ragas_version": "0.2.5",
+  "original_hash": -1698100170803872933,
+  "language": "chinese",
+  "instruction": "根据指定的条件（角色、主题、风格、长度）和提供的上下文生成查询和答案。确保答案完全忠实于上下文，仅使用直接来自提供节点的信息。### 指令：\n1. **生成查询**：根据上下文、角色、主题、风格和长度，创建一个与角色视角一致并反映主题的问题。\n2. **生成答案**：仅使用提供的上下文内容，创建一个忠实且详细的答案。不要包含任何不在或无法从给定上下文中推断的信息。\n### 示例输出：\n\n",
+  "examples": []
+}

evalscope/backend/rag_eval/ragas/prompts/chinese/MultiHopSpecificQuerySynthesizer/theme_persona_matching_prompt_chinese.json

@@ -0,0 +1,39 @@
+{
+  "ragas_version": "0.2.5",
+  "original_hash": 2334929353739018813,
+  "language": "chinese",
+  "instruction": "给定一个主题和角色列表，根据角色描述将每个角色与相关主题关联起来。",
+  "examples": [
+    {
+      "input": {
+        "themes": [
+          "同理心",
+          "包容性",
+          "远程工作"
+        ],
+        "personas": [
+          {
+            "name": "人力资源经理",
+            "role_description": "专注于包容性和员工支持。"
+          },
+          {
+            "name": "远程团队负责人",
+            "role_description": "管理远程团队沟通。"
+          }
+        ]
+      },
+      "output": {
+        "mapping": {
+          "HR Manager": [
+            "包容性",
+            "同理心"
+          ],
+          "Remote Team Lead": [
+            "远程工作",
+            "同理心"
+          ]
+        }
+      }
+    }
+  ]
+}

evalscope/backend/rag_eval/ragas/prompts/chinese/MultiModalFaithfulness/faithfulness_prompt_chinese.json

@@ -0,0 +1,34 @@
+{
+  "ragas_version": "0.2.5",
+  "original_hash": -2189588237940965149,
+  "language": "chinese",
+  "instruction": "请说明给定的信息是否得到视觉和文本上下文信息的支持。您需要回答“是”或“否”。如果任何图像和文本上下文支持该信息，请回答“是”。",
+  "examples": [
+    {
+      "input": {
+        "response": "苹果派通常是双层皮的。",
+        "retrieved_contexts": [
+          "苹果派是一种水果派，其主要馅料成分是苹果。",
+          "苹果派通常与奶油、冰淇淋（“苹果派 à la mode”）、蛋奶沙司或切达干酪一起食用。",
+          "它通常是双层皮的，上下都有糕点；上层皮可以是实心的或格状的（交叉条纹编织而成）。"
+        ]
+      },
+      "output": {
+        "faithful": true
+      }
+    },
+    {
+      "input": {
+        "response": "苹果派味道不好。",
+        "retrieved_contexts": [
+          "苹果派是一种水果派，其主要馅料成分是苹果。",
+          "苹果派通常与奶油、冰淇淋（“苹果派 à la mode”）、蛋奶沙司或切达干酪一起食用。",
+          "它通常是双层皮的，上下都有糕点；上层皮可以是实心的或格状的（交叉条纹编织而成）。"
+        ]
+      },
+      "output": {
+        "faithful": false
+      }
+    }
+  ]
+}

evalscope/backend/rag_eval/ragas/prompts/chinese/MultiModalRelevance/relevance_prompt_chinese.json

@@ -0,0 +1,36 @@
+{
+  "ragas_version": "0.2.5",
+  "original_hash": -7302860412443151372,
+  "language": "chinese",
+  "instruction": "\n您的任务是评估查询的响应是否与提供的图像和文本上下文信息一致。\n您有两个选项可以回答。要么是 True / False。\n如果查询的响应与上下文信息一致，则回答 - True，否则为 False。\n",
+  "examples": [
+    {
+      "input": {
+        "user_input": "传统玛格丽塔披萨的主要成分是什么？",
+        "response": "玛格丽塔披萨的主要成分是番茄、马苏里拉奶酪和新鲜罗勒。",
+        "retrieved_contexts": [
+          "传统的玛格丽塔披萨由薄薄的饼皮组成。",
+          "主要的配料包括番茄、马苏里拉奶酪、新鲜罗勒、盐和橄榄油。",
+          "它是最简单和最经典的披萨类型之一。"
+        ]
+      },
+      "output": {
+        "relevance": true
+      }
+    },
+    {
+      "input": {
+        "user_input": "谁在2021年奥斯卡颁奖典礼上获得了最佳男演员奖？",
+        "response": "2021年的最佳男演员奖由莱昂纳多·迪卡普里奥获得。",
+        "retrieved_contexts": [
+          "第93届奥斯卡颁奖典礼于2021年举行。",
+          "安东尼·霍普金斯凭借在《困在时间里的父亲》中的角色赢得了最佳男演员奖。",
+          "由于COVID-19的限制，这次活动具有独特性。"
+        ]
+      },
+      "output": {
+        "relevance": false
+      }
+    }
+  ]
+}

evalscope/backend/rag_eval/ragas/prompts/chinese/NERExtractor/prompt_chinese.json

@@ -0,0 +1,25 @@
+{
+  "ragas_version": "0.2.5",
+  "original_hash": -7036736759899743798,
+  "language": "chinese",
+  "instruction": "从给定文本中提取命名实体，限制输出为最重要的实体。确保实体数量不超过指定的最大值。",
+  "examples": [
+    {
+      "input": {
+        "text": "特斯拉和SpaceX的首席执行官埃隆·马斯克宣布计划将业务扩展到欧洲和亚洲的新地点。\n                此次扩展预计将创造数千个就业机会，特别是在柏林和上海等城市。",
+        "max_num": 10
+      },
+      "output": {
+        "entities": [
+          "埃隆·马斯克",
+          "特斯拉",
+          "SpaceX",
+          "欧洲",
+          "亚洲",
+          "柏林",
+          "上海"
+        ]
+      }
+    }
+  ]
+}

evalscope/backend/rag_eval/ragas/prompts/chinese/SingleHopSpecificQuerySynthesizer/generate_query_reference_prompt_chinese.json

@@ -0,0 +1,7 @@
+{
+  "ragas_version": "0.2.5",
+  "original_hash": -1422723613754983378,
+  "language": "chinese",
+  "instruction": "根据指定的条件（角色、术语、风格、长度）和提供的上下文生成查询和答案。确保答案完全忠实于上下文，仅使用直接来自提供上下文的信息。### 指令：\n1. **生成查询**：根据上下文、角色、术语、风格和长度，创建一个与角色视角一致并包含术语的问题。\n2. **生成答案**：仅使用提供的上下文中的内容，构建对查询的详细答案。不要添加上下文中未包含或无法推断的信息。\n### 示例输出：\n\n",
+  "examples": []
+}

evalscope/backend/rag_eval/ragas/prompts/chinese/SingleHopSpecificQuerySynthesizer/theme_persona_matching_prompt_chinese.json

@@ -0,0 +1,39 @@
+{
+  "ragas_version": "0.2.5",
+  "original_hash": 2334929353739018813,
+  "language": "chinese",
+  "instruction": "给定一个主题和角色列表，根据角色描述将每个角色与相关主题关联起来。",
+  "examples": [
+    {
+      "input": {
+        "themes": [
+          "同理心",
+          "包容性",
+          "远程工作"
+        ],
+        "personas": [
+          {
+            "name": "人力资源经理",
+            "role_description": "专注于包容性和员工支持。"
+          },
+          {
+            "name": "远程团队领导",
+            "role_description": "管理远程团队沟通。"
+          }
+        ]
+      },
+      "output": {
+        "mapping": {
+          "HR Manager": [
+            "包容性",
+            "同理心"
+          ],
+          "Remote Team Lead": [
+            "远程工作",
+            "同理心"
+          ]
+        }
+      }
+    }
+  ]
+}

evalscope/backend/rag_eval/ragas/prompts/chinese/SummaryExtractor/prompt_chinese.json

@@ -0,0 +1,16 @@
+{
+  "ragas_version": "0.2.5",
+  "original_hash": -5467318232123540806,
+  "language": "chinese",
+  "instruction": "将给定文本总结为少于10个句子。",
+  "examples": [
+    {
+      "input": {
+        "text": "人工智能\n\n人工智能正在通过自动化以前需要人类智能的任务来改变各个行业。从医疗到金融，人工智能正在被用来快速准确地分析大量数据。这项技术还推动了自动驾驶汽车和个性化推荐等领域的创新。"
+      },
+      "output": {
+        "text": "人工智能通过自动化任务、分析数据和推动自动驾驶汽车和个性化推荐等创新，正在革新各个行业。"
+      }
+    }
+  ]
+}

evalscope/backend/rag_eval/ragas/prompts/chinese/ThemesExtractor/prompt_chinese.json

@@ -0,0 +1,24 @@
+{
+  "ragas_version": "0.2.5",
+  "original_hash": 2452110859551524285,
+  "language": "chinese",
+  "instruction": "从给定的文本中提取主要主题和概念。",
+  "examples": [
+    {
+      "input": {
+        "text": "人工智能通过自动化需要人类智能的任务来改变行业。人工智能快速准确地分析大量数据，推动了自动驾驶汽车和个性化推荐等创新。",
+        "max_num": 10
+      },
+      "output": {
+        "output": [
+          "人工智能",
+          "自动化",
+          "数据分析",
+          "创新",
+          "自动驾驶汽车",
+          "个性化推荐"
+        ]
+      }
+    }
+  ]
+}

evalscope/backend/rag_eval/ragas/prompts/persona_prompt.py

@@ -0,0 +1,18 @@
+import typing as t
+
+from pydantic import BaseModel
+from ragas.prompt import PydanticPrompt, StringIO
+from ragas.testset.persona import Persona
+
+
+class PersonaGenerationPromptZH(PydanticPrompt[StringIO, Persona]):
+    instruction: str = ('使用提供的摘要，生成一个可能会与内容互动或从中受益的角色。包括一个独特的名字和一个简洁的角色描述。')
+    input_model: t.Type[StringIO] = StringIO
+    output_model: t.Type[Persona] = Persona
+    examples: t.List[t.Tuple[StringIO, Persona]] = [(
+        StringIO(text='《数字营销指南》解释了在各种在线平台上吸引受众的策略。'),
+        Persona(
+            name='数字营销专家',
+            role_description='专注于吸引受众并在线上提升品牌。',
+        ),
+    )]

evalscope/backend/rag_eval/ragas/tasks/testset_generation.py

@@ -1,15 +1,15 @@
-import os
 import asyncio
+import os
+
 import pandas as pd
-from tqdm import tqdm
-from ragas.llms import LangchainLLMWrapper
 from ragas.embeddings import LangchainEmbeddingsWrapper
-from .translate_prompt import translate_prompts
-from evalscope.utils.logger import get_logger
-from evalscope.backend.rag_eval.ragas.arguments import TestsetGenerationArguments
-from evalscope.backend.rag_eval import EmbeddingModel, LLM, ChatOpenAI
+from ragas.llms import LangchainLLMWrapper
+from tqdm import tqdm
 
-os.environ['DO_NOT_TRACK'] = 'true'
+from evalscope.backend.rag_eval import LLM, ChatOpenAI, EmbeddingModel
+from evalscope.backend.rag_eval.ragas.arguments import TestsetGenerationArguments
+from evalscope.utils.logger import get_logger
+from .translate_prompt import translate_prompts
 
 logger = get_logger()
 
@@ -17,116 +17,110 @@ logger = get_logger()
 def get_transform(llm, embedding, language):
     """
     Creates and returns a default set of transforms for processing a knowledge graph.
-
-    This function defines a series of transformation steps to be applied to a
-    knowledge graph, including extracting summaries, keyphrases, titles,
-    headlines, and embeddings, as well as building similarity relationships
-    between nodes.
-
-    The transforms are applied in the following order:
-    1. Parallel extraction of summaries and headlines
-    2. Embedding of summaries for document nodes
-    3. Splitting of headlines
-    4. Parallel extraction of embeddings, keyphrases, and titles
-    5. Building cosine similarity relationships between nodes
-    6. Building cosine similarity relationships between summaries
-
-    Returns
-    -------
-    Transforms
-        A list of transformation steps to be applied to the knowledge graph.
-
     """
     from ragas.testset.transforms.engine import Parallel
     from ragas.testset.transforms.extractors import (
         EmbeddingExtractor,
         HeadlinesExtractor,
-        KeyphrasesExtractor,
         SummaryExtractor,
-        TitleExtractor,
     )
-    from ragas.testset.transforms.relationship_builders.cosine import (
+    from ragas.testset.transforms.extractors.llm_based import NERExtractor, ThemesExtractor
+    from ragas.testset.transforms.relationship_builders import (
         CosineSimilarityBuilder,
-        SummaryCosineSimilarityBuilder,
+        OverlapScoreBuilder,
     )
     from ragas.testset.transforms.splitters import HeadlineSplitter
+    from ragas.testset.transforms.filters import CustomNodeFilter
     from ragas.testset.graph import NodeType
+    from ragas.utils import num_tokens_from_string
+
+    def summary_filter(node):
+        return (node.type == NodeType.DOCUMENT and num_tokens_from_string(node.properties['page_content']) > 500)
 
-    # define the transforms
-    summary_extractor = SummaryExtractor(llm=llm)
-    keyphrase_extractor = KeyphrasesExtractor(llm=llm)
-    title_extractor = TitleExtractor(llm=llm)
+    summary_extractor = SummaryExtractor(llm=llm, filter_nodes=lambda node: summary_filter(node))
+    ner_extractor = NERExtractor(llm=llm, filter_nodes=lambda node: node.type == NodeType.CHUNK)
+    theme_extractor = ThemesExtractor(llm=llm)
     headline_extractor = HeadlinesExtractor(llm=llm)
 
     asyncio.run(
         translate_prompts(
             prompts=[
                 summary_extractor,
-                keyphrase_extractor,
-                title_extractor,
+                theme_extractor,
+                ner_extractor,
                 headline_extractor,
             ],
             target_lang=language,
             llm=llm,
             adapt_instruction=True,
-        )
-    )
+        ))
+
+    splitter = HeadlineSplitter(min_tokens=500)
 
-    embedding_extractor = EmbeddingExtractor(embedding_model=embedding)
-    headline_splitter = HeadlineSplitter()
-    cosine_sim_builder = CosineSimilarityBuilder(threshold=0.8)
-    summary_embedder = EmbeddingExtractor(
-        name='summary_embedder',
-        filter_nodes=lambda node: True if node.type == NodeType.DOCUMENT else False,
+    summary_emb_extractor = EmbeddingExtractor(
+        embedding_model=embedding,
         property_name='summary_embedding',
         embed_property_name='summary',
-        embedding_model=embedding,
+        filter_nodes=lambda node: summary_filter(node),
     )
-    summary_cosine_sim_builder = SummaryCosineSimilarityBuilder(threshold=0.6)
 
-    # specify the transforms and their order to be applied
+    cosine_sim_builder = CosineSimilarityBuilder(
+        property_name='summary_embedding',
+        new_property_name='summary_similarity',
+        threshold=0.7,
+        filter_nodes=lambda node: summary_filter(node),
+    )
+
+    ner_overlap_sim = OverlapScoreBuilder(threshold=0.01, filter_nodes=lambda node: node.type == NodeType.CHUNK)
+
+    node_filter = CustomNodeFilter(llm=llm, filter_nodes=lambda node: node.type == NodeType.CHUNK)
+
     transforms = [
-        Parallel(summary_extractor, headline_extractor),
-        summary_embedder,
-        headline_splitter,
-        Parallel(embedding_extractor, keyphrase_extractor, title_extractor),
-        cosine_sim_builder,
-        summary_cosine_sim_builder,
+        headline_extractor,
+        splitter,
+        summary_extractor,
+        node_filter,
+        Parallel(summary_emb_extractor, theme_extractor, ner_extractor),
+        Parallel(cosine_sim_builder, ner_overlap_sim),
     ]
+
     return transforms
 
 
 def get_distribution(llm, distribution, language):
-    from ragas.testset.synthesizers.abstract_query import (
-        AbstractQuerySynthesizer,
-        ComparativeAbstractQuerySynthesizer,
+    from ragas.testset.synthesizers.multi_hop import (
+        MultiHopAbstractQuerySynthesizer,
+        MultiHopSpecificQuerySynthesizer,
     )
-    from ragas.testset.synthesizers.specific_query import SpecificQuerySynthesizer
+    from ragas.testset.synthesizers.single_hop.specific import (
+        SingleHopSpecificQuerySynthesizer, )
 
-    abstract = AbstractQuerySynthesizer(llm=llm)
-    comparative = ComparativeAbstractQuerySynthesizer(llm=llm)
-    specific = SpecificQuerySynthesizer(llm=llm)
+    single_hop = SingleHopSpecificQuerySynthesizer(llm=llm)
+    multi_hop_abs = MultiHopAbstractQuerySynthesizer(llm=llm)
+    multi_hop_spec = MultiHopSpecificQuerySynthesizer(llm=llm)
 
     asyncio.run(
         translate_prompts(
             prompts=[
-                abstract,
-                comparative,
-                specific,
+                single_hop,
+                multi_hop_abs,
+                multi_hop_spec,
             ],
             target_lang=language,
             llm=llm,
             adapt_instruction=True,
-        )
-    )
-    return [
-        (abstract, distribution['simple']),
-        (comparative, distribution['multi_context']),
-        (specific, distribution['reasoning']),
-    ]
+        ))
+
+    mapping = {
+        'simple': single_hop,
+        'multi_context': multi_hop_abs,
+        'reasoning': multi_hop_spec,
+    }
+
+    return [(mapping[key], distribution[key]) for key in mapping if key in distribution]
 
 
-def get_knowledge_graph(documents, transforms, local_file):
+def get_knowledge_graph(documents, transforms, local_file, run_config):
     from ragas.testset.graph import KnowledgeGraph, Node, NodeType
     from ragas.testset.transforms import apply_transforms
 
@@ -148,7 +142,7 @@ def get_knowledge_graph(documents, transforms, local_file):
     kg = KnowledgeGraph(nodes=nodes)
 
     # apply transforms and update the knowledge graph
-    apply_transforms(kg, transforms)
+    apply_transforms(kg, transforms, run_config=run_config)
 
     # save the knowledge graph
     output_path = os.path.dirname(local_file)
@@ -158,6 +152,39 @@ def get_knowledge_graph(documents, transforms, local_file):
     return kg
 
 
+def get_persona(llm, kg, language):
+    from evalscope.backend.rag_eval.ragas.prompts.persona_prompt import PersonaGenerationPromptZH
+    from ragas.testset.persona import generate_personas_from_kg, PersonaGenerationPrompt
+    from ragas.testset.graph import Node
+
+    def filter(node: Node) -> bool:
+        if (node.type.name == 'DOCUMENT' and node.properties.get('summary_embedding') is not None):
+            return True
+        else:
+            return False
+
+    if language == 'chinese':
+        persona_prompt = PersonaGenerationPromptZH()
+    else:
+        persona_prompt = PersonaGenerationPrompt()
+    # NOTE: can't translate this yet
+    # asyncio.run(
+    #     translate_prompts(
+    #         prompts=[persona_prompt],
+    #         target_lang=language,
+    #         llm=llm,
+    #         adapt_instruction=True,
+    #     ))
+
+    return generate_personas_from_kg(
+        llm=llm,
+        kg=kg,
+        num_personas=3,
+        persona_generation_prompt=persona_prompt,
+        filter_fn=filter,
+    )
+
+
 def load_data(file_path):
     from langchain_community.document_loaders import UnstructuredFileLoader
 
@@ -178,32 +205,31 @@ def generate_testset(args: TestsetGenerationArguments) -> None:
     generator_llm = LLM.load(**args.generator_llm)
     embeddings = EmbeddingModel.load(**args.embeddings)
 
+    wrapped_llm = LangchainLLMWrapper(generator_llm)
+    wrapped_embeddings = LangchainEmbeddingsWrapper(embeddings)
+
     # Change resulting question type distribution
-    distributions = get_distribution(
-        LangchainLLMWrapper(generator_llm), args.distribution, args.language
-    )
+    distributions = get_distribution(wrapped_llm, args.distribution, args.language)
 
+    run_config = RunConfig(timeout=600, max_retries=3, max_wait=120, max_workers=1, log_tenacity=True)
     # get transforms
     transforms = get_transform(
-        LangchainLLMWrapper(generator_llm),
-        LangchainEmbeddingsWrapper(embeddings),
+        wrapped_llm,
+        wrapped_embeddings,
         args.language,
     )
 
     # get knowledge graph
-    knowledge_graph = get_knowledge_graph(documents, transforms, args.knowledge_graph)
+    knowledge_graph = get_knowledge_graph(documents, transforms, args.knowledge_graph, run_config)
 
-    generator = TestsetGenerator.from_langchain(
-        generator_llm, embeddings, knowledge_graph
-    )
+    persona_list = get_persona(llm=wrapped_llm, kg=knowledge_graph, language=args.language)
+
+    generator = TestsetGenerator(llm=wrapped_llm, knowledge_graph=knowledge_graph, persona_list=persona_list)
 
-    runconfig = RunConfig(
-        timeout=600, max_retries=3, max_wait=120, max_workers=1, log_tenacity=True
-    )
     testset = generator.generate(
         testset_size=args.test_size,
         query_distribution=distributions,
-        run_config=runconfig,
+        run_config=run_config,
         with_debugging_logs=True,
         raise_exceptions=True,
     )
@@ -212,9 +238,7 @@ def generate_testset(args: TestsetGenerationArguments) -> None:
     testset_df = testset.to_pandas()
     output_path = os.path.dirname(args.output_file)
     os.makedirs(output_path, exist_ok=True)
-    testset_df.to_json(
-        args.output_file, indent=4, index=False, orient='records', force_ascii=False
-    )
+    testset_df.to_json(args.output_file, indent=4, index=False, orient='records', force_ascii=False)
 
     # get answer
     testset_with_answer = get_answer(testset_df, generator_llm, args.language)
@@ -243,21 +267,17 @@ Answer:
         contexts = '\n'.join(row['reference_contexts'])
 
         # Combine question and contexts as input for the LLM
-        input_text = template.format(
-            language=language, question=question, contexts=contexts
-        )
+        input_text = template.format(language=language, question=question, contexts=contexts)
 
         # Generate the answer using the generator LLM
         answer = generator_llm.invoke(input_text)
         if isinstance(generator_llm, ChatOpenAI):
             answer = answer.content
-        items.append(
-            {
-                'user_input': question,
-                'retrieved_contexts': row['reference_contexts'],
-                'response': answer,
-                'reference': row['reference'],
-            }
-        )
+        items.append({
+            'user_input': question,
+            'retrieved_contexts': row['reference_contexts'],
+            'response': answer,
+            'reference': row['reference'],
+        })
 
     return pd.DataFrame.from_dict(items)