evalscope 0.5.5rc1__py3-none-any.whl → 0.6.0__py3-none-any.whl

evalscope/backend/rag_eval/cmteb/tasks/STS.py

@@ -0,0 +1,302 @@
+from mteb.abstasks.AbsTaskSTS import AbsTaskSTS
+from mteb.abstasks.TaskMetadata import TaskMetadata
+
+class ATEC(AbsTaskSTS):
+    metadata = TaskMetadata(
+        name="ATEC",
+        dataset={
+            "path": "C-MTEB/ATEC",
+            "revision": "0f319b1142f28d00e055a6770f3f726ae9b7d865",
+        },
+        description="A Chinese dataset for textual relatedness",
+        reference="https://aclanthology.org/2021.emnlp-main.357",
+        type="STS",
+        category="s2s",
+        modalities=["text"],
+        eval_splits=["validation", "test"],
+        eval_langs=["cmn-Hans"],
+        main_score="cosine_spearman",
+        date=None,
+        domains=None,
+        task_subtypes=None,
+        license=None,
+        annotations_creators=None,
+        dialect=None,
+        sample_creation=None,
+        bibtex_citation="""@inproceedings{raghu-etal-2021-end,
+    title = "End-to-End Learning of Flowchart Grounded Task-Oriented Dialogs",
+    author = "Raghu, Dinesh  and
+      Agarwal, Shantanu  and
+      Joshi, Sachindra  and
+      {Mausam}",
+    editor = "Moens, Marie-Francine  and
+      Huang, Xuanjing  and
+      Specia, Lucia  and
+      Yih, Scott Wen-tau",
+    booktitle = "Proceedings of the 2021 Conference on Empirical Methods in Natural Language Processing",
+    month = nov,
+    year = "2021",
+    address = "Online and Punta Cana, Dominican Republic",
+    publisher = "Association for Computational Linguistics",
+    url = "https://aclanthology.org/2021.emnlp-main.357",
+    doi = "10.18653/v1/2021.emnlp-main.357",
+    pages = "4348--4366",
+    abstract = "We propose a novel problem within end-to-end learning of task oriented dialogs (TOD), in which the dialog system mimics a troubleshooting agent who helps a user by diagnosing their problem (e.g., car not starting). Such dialogs are grounded in domain-specific flowcharts, which the agent is supposed to follow during the conversation. Our task exposes novel technical challenges for neural TOD, such as grounding an utterance to the flowchart without explicit annotation, referring to additional manual pages when user asks a clarification question, and ability to follow unseen flowcharts at test time. We release a dataset (FLODIAL) consisting of 2,738 dialogs grounded on 12 different troubleshooting flowcharts. We also design a neural model, FLONET, which uses a retrieval-augmented generation architecture to train the dialog agent. Our experiments find that FLONET can do zero-shot transfer to unseen flowcharts, and sets a strong baseline for future research.",
+}""",
+        descriptive_stats={"n_samples": None, "avg_character_length": None},
+    )
+
+    @property
+    def metadata_dict(self) -> dict[str, str]:
+        metadata_dict = super().metadata_dict
+        metadata_dict["min_score"] = 0
+        metadata_dict["max_score"] = 1
+        return metadata_dict
+
+
+class BQ(AbsTaskSTS):
+    metadata = TaskMetadata(
+        name="BQ",
+        dataset={
+            "path": "C-MTEB/BQ",
+            "revision": "e3dda5e115e487b39ec7e618c0c6a29137052a55",
+        },
+        description="A Chinese dataset for textual relatedness",
+        reference="https://aclanthology.org/2021.emnlp-main.357",
+        type="STS",
+        category="s2s",
+        modalities=["text"],
+        eval_splits=["validation", "test"],
+        eval_langs=["cmn-Hans"],
+        main_score="cosine_spearman",
+        date=None,
+        domains=None,
+        task_subtypes=None,
+        license=None,
+        annotations_creators=None,
+        dialect=None,
+        sample_creation=None,
+        bibtex_citation="""@misc{xiao2024cpackpackagedresourcesadvance,
+      title={C-Pack: Packaged Resources To Advance General Chinese Embedding}, 
+      author={Shitao Xiao and Zheng Liu and Peitian Zhang and Niklas Muennighoff and Defu Lian and Jian-Yun Nie},
+      year={2024},
+      eprint={2309.07597},
+      archivePrefix={arXiv},
+      primaryClass={cs.CL},
+      url={https://arxiv.org/abs/2309.07597}, 
+}""",
+        descriptive_stats={"n_samples": None, "avg_character_length": None},
+    )
+
+    @property
+    def metadata_dict(self) -> dict[str, str]:
+        metadata_dict = super().metadata_dict
+        metadata_dict["min_score"] = 0
+        metadata_dict["max_score"] = 1
+        return metadata_dict
+
+
+class LCQMC(AbsTaskSTS):
+    metadata = TaskMetadata(
+        name="LCQMC",
+        dataset={
+            "path": "C-MTEB/LCQMC",
+            "revision": "17f9b096f80380fce5ed12a9be8be7784b337daf",
+        },
+        description="A Chinese dataset for textual relatedness",
+        reference="https://aclanthology.org/2021.emnlp-main.357",
+        type="STS",
+        category="s2s",
+        modalities=["text"],
+        eval_splits=["test"],
+        eval_langs=["cmn-Hans"],
+        main_score="cosine_spearman",
+        date=None,
+        domains=None,
+        task_subtypes=None,
+        license=None,
+        annotations_creators=None,
+        dialect=None,
+        sample_creation=None,
+        bibtex_citation="""@misc{xiao2024cpackpackagedresourcesadvance,
+      title={C-Pack: Packaged Resources To Advance General Chinese Embedding}, 
+      author={Shitao Xiao and Zheng Liu and Peitian Zhang and Niklas Muennighoff and Defu Lian and Jian-Yun Nie},
+      year={2024},
+      eprint={2309.07597},
+      archivePrefix={arXiv},
+      primaryClass={cs.CL},
+      url={https://arxiv.org/abs/2309.07597}, 
+}""",
+        descriptive_stats={"n_samples": None, "avg_character_length": None},
+    )
+
+    @property
+    def metadata_dict(self) -> dict[str, str]:
+        metadata_dict = super().metadata_dict
+        metadata_dict["min_score"] = 0
+        metadata_dict["max_score"] = 1
+        return metadata_dict
+
+
+class PAWSX(AbsTaskSTS):
+    metadata = TaskMetadata(
+        name="PAWSX",
+        dataset={
+            "path": "C-MTEB/PAWSX",
+            "revision": "9c6a90e430ac22b5779fb019a23e820b11a8b5e1",
+        },
+        description="A Chinese dataset for textual relatedness",
+        reference="https://aclanthology.org/2021.emnlp-main.357",
+        type="STS",
+        category="s2s",
+        modalities=["text"],
+        eval_splits=["test"],
+        eval_langs=["cmn-Hans"],
+        main_score="cosine_spearman",
+        date=None,
+        domains=None,
+        task_subtypes=None,
+        license=None,
+        annotations_creators=None,
+        dialect=None,
+        sample_creation=None,
+        bibtex_citation="""@misc{xiao2024cpackpackagedresourcesadvance,
+      title={C-Pack: Packaged Resources To Advance General Chinese Embedding}, 
+      author={Shitao Xiao and Zheng Liu and Peitian Zhang and Niklas Muennighoff and Defu Lian and Jian-Yun Nie},
+      year={2024},
+      eprint={2309.07597},
+      archivePrefix={arXiv},
+      primaryClass={cs.CL},
+      url={https://arxiv.org/abs/2309.07597}, 
+}""",
+        descriptive_stats={"n_samples": None, "avg_character_length": None},
+    )
+
+    @property
+    def metadata_dict(self) -> dict[str, str]:
+        metadata_dict = super().metadata_dict
+        metadata_dict["min_score"] = 0
+        metadata_dict["max_score"] = 1
+        return metadata_dict
+
+
+class STSB(AbsTaskSTS):
+    metadata = TaskMetadata(
+        name="STSB",
+        dataset={
+            "path": "C-MTEB/STSB",
+            "revision": "0cde68302b3541bb8b3c340dc0644b0b745b3dc0",
+        },
+        description="A Chinese dataset for textual relatedness",
+        reference="https://aclanthology.org/2021.emnlp-main.357",
+        type="STS",
+        category="s2s",
+        modalities=["text"],
+        eval_splits=["validation", "test"],
+        eval_langs=["cmn-Hans"],
+        main_score="cosine_spearman",
+        date=None,
+        domains=None,
+        task_subtypes=None,
+        license=None,
+        annotations_creators=None,
+        dialect=None,
+        sample_creation=None,
+        bibtex_citation="""@misc{xiao2024cpackpackagedresourcesadvance,
+      title={C-Pack: Packaged Resources To Advance General Chinese Embedding}, 
+      author={Shitao Xiao and Zheng Liu and Peitian Zhang and Niklas Muennighoff and Defu Lian and Jian-Yun Nie},
+      year={2024},
+      eprint={2309.07597},
+      archivePrefix={arXiv},
+      primaryClass={cs.CL},
+      url={https://arxiv.org/abs/2309.07597}, 
+}""",
+        descriptive_stats={"n_samples": None, "avg_character_length": None},
+    )
+
+    @property
+    def metadata_dict(self) -> dict[str, str]:
+        metadata_dict = super().metadata_dict
+        metadata_dict["min_score"] = 0
+        metadata_dict["max_score"] = 5
+        return metadata_dict
+
+
+class AFQMC(AbsTaskSTS):
+    metadata = TaskMetadata(
+        name="AFQMC",
+        dataset={
+            "path": "C-MTEB/AFQMC",
+            "revision": "b44c3b011063adb25877c13823db83bb193913c4",
+        },
+        description="A Chinese dataset for textual relatedness",
+        reference="https://aclanthology.org/2021.emnlp-main.357",
+        type="STS",
+        category="s2s",
+        modalities=["text"],
+        eval_splits=["validation"],
+        eval_langs=["cmn-Hans"],
+        main_score="cosine_spearman",
+        date=None,
+        domains=None,
+        task_subtypes=None,
+        license=None,
+        annotations_creators=None,
+        dialect=None,
+        sample_creation=None,
+        bibtex_citation="""@inproceedings{raghu-etal-2021-end,
+    title = "End-to-End Learning of Flowchart Grounded Task-Oriented Dialogs",
+    author = "Raghu, Dinesh  and
+      Agarwal, Shantanu  and
+      Joshi, Sachindra  and
+      {Mausam}",
+    editor = "Moens, Marie-Francine  and
+      Huang, Xuanjing  and
+      Specia, Lucia  and
+      Yih, Scott Wen-tau",
+    booktitle = "Proceedings of the 2021 Conference on Empirical Methods in Natural Language Processing",
+    month = nov,
+    year = "2021",
+    address = "Online and Punta Cana, Dominican Republic",
+    publisher = "Association for Computational Linguistics",
+    url = "https://aclanthology.org/2021.emnlp-main.357",
+    doi = "10.18653/v1/2021.emnlp-main.357",
+    pages = "4348--4366",
+    abstract = "We propose a novel problem within end-to-end learning of task oriented dialogs (TOD), in which the dialog system mimics a troubleshooting agent who helps a user by diagnosing their problem (e.g., car not starting). Such dialogs are grounded in domain-specific flowcharts, which the agent is supposed to follow during the conversation. Our task exposes novel technical challenges for neural TOD, such as grounding an utterance to the flowchart without explicit annotation, referring to additional manual pages when user asks a clarification question, and ability to follow unseen flowcharts at test time. We release a dataset (FLODIAL) consisting of 2,738 dialogs grounded on 12 different troubleshooting flowcharts. We also design a neural model, FLONET, which uses a retrieval-augmented generation architecture to train the dialog agent. Our experiments find that FLONET can do zero-shot transfer to unseen flowcharts, and sets a strong baseline for future research.",
+}""",
+        descriptive_stats={"n_samples": None, "avg_character_length": None},
+    )
+
+    @property
+    def metadata_dict(self) -> dict[str, str]:
+        metadata_dict = super().metadata_dict
+        metadata_dict["min_score"] = 0
+        metadata_dict["max_score"] = 1
+        return metadata_dict
+
+
+class QBQTC(AbsTaskSTS):
+    metadata = TaskMetadata(
+        name="QBQTC",
+        dataset={
+            "path": "C-MTEB/QBQTC",
+            "revision": "790b0510dc52b1553e8c49f3d2afb48c0e5c48b7",
+        },
+        description="",
+        reference="https://github.com/CLUEbenchmark/QBQTC/tree/main/dataset",
+        type="STS",
+        category="s2s",
+        modalities=["text"],
+        eval_splits=["test"],
+        eval_langs=["cmn-Hans"],
+        main_score="cosine_spearman",
+        date=None,
+        domains=None,
+        task_subtypes=None,
+        license=None,
+        annotations_creators=None,
+        dialect=None,
+        sample_creation=None,
+        bibtex_citation=None,
+        descriptive_stats={"n_samples": None, "avg_character_length": None},
+    )

evalscope/backend/rag_eval/cmteb/tasks/__init__.py

@@ -0,0 +1,70 @@
+from .Classification import *
+from .Clustering import *
+from .PairClassification import *
+from .Reranking import *
+from .Retrieval import *
+from .STS import *
+from .CustomTask import *
+
+
+CLS_CLASSIFICATION = {
+    "TNews": TNews,
+    "IFlyTek": IFlyTek,
+    "MultilingualSentiment": MultilingualSentiment,
+    "JDReview": JDReview,
+    "OnlineShopping": OnlineShopping,
+    "Waimai": Waimai,
+}
+
+CLS_CLUSTERING = {
+    "CLSClusteringS2S": CLSClusteringFastS2S,
+    "CLSClusteringP2P": CLSClusteringFastP2P,
+    "ThuNewsClusteringS2S": ThuNewsClusteringFastS2S,
+    "ThuNewsClusteringP2P": ThuNewsClusteringFastP2P,
+}
+
+CLS_PAIR_CLASSIFICATION = {
+    "Ocnli": Ocnli,
+    "Cmnli": Cmnli,
+}
+
+CLS_RERANKING = {
+    "T2Reranking": T2Reranking,
+    "MMarcoReranking": MMarcoReranking,
+    "CMedQAv1": CMedQAv1,
+    "CMedQAv2": CMedQAv2,
+}
+
+CLS_RETRIEVAL = {
+    "T2Retrieval": T2Retrieval,
+    "MMarcoRetrieval": MMarcoRetrieval,
+    "DuRetrieval": DuRetrieval,
+    "CovidRetrieval": CovidRetrieval,
+    "CmedqaRetrieval": CmedqaRetrieval,
+    "EcomRetrieval": EcomRetrieval,
+    "MedicalRetrieval": MedicalRetrieval,
+    "VideoRetrieval": VideoRetrieval,
+}
+
+CLS_STS = {
+    "ATEC": ATEC,
+    "BQ": BQ,
+    "LCQMC": LCQMC,
+    "PAWSX": PAWSX,
+    "STSB": STSB,
+    "AFQMC": AFQMC,
+    "QBQTC": QBQTC,
+}
+
+CLS_CUSTOM = {
+    "CustomRetrieval": CustomRetrieval,
+}
+
+CLS_DICT = {
+    **CLS_CLASSIFICATION,
+    **CLS_CLUSTERING,
+    **CLS_PAIR_CLASSIFICATION,
+    **CLS_RERANKING,
+    **CLS_RETRIEVAL,
+    **CLS_STS,
+}

evalscope/backend/rag_eval/ragas/__init__.py

@@ -0,0 +1,2 @@
+from evalscope.backend.rag_eval.ragas.arguments import TestsetGenerationArguments, EvaluationArguments
+from evalscope.backend.rag_eval.ragas.task_template import rag_eval

evalscope/backend/rag_eval/ragas/arguments.py

@@ -0,0 +1,47 @@
+from dataclasses import dataclass, field
+from typing import List, Optional, Union, Dict, Any
+
+
+@dataclass
+class TestsetGenerationArguments:
+    docs: List[str] = field(default_factory=list)
+    test_size: int = 10
+    output_file: str = 'outputs/testset.json'
+    knowledge_graph: str = 'outputs/knowledge_graph.json'
+    """
+    For local LLM support, you can use the following fields:
+        model_name_or_path: str
+        model_revision: str = "master"
+        template_type: str = "default"
+        generation_config: Optional[Dict]
+
+    For API LLM support, you can use the following fields:
+        model_name="gpt-4o-mini"
+        api_base: str = "",
+        api_key: Optional[str] = None
+    """
+    generator_llm: Dict = field(default_factory=dict)
+    embeddings: Dict = field(default_factory=dict)
+    distribution: str = field(
+        default_factory=lambda: {'simple': 0.5, 'multi_context': 0.4, 'reasoning': 0.1}
+    )
+    # For LLM based evaluation
+    # available: ['english', 'hindi', 'marathi', 'chinese', 'spanish', 'amharic', 'arabic',
+    # 'armenian', 'bulgarian', 'urdu', 'russian', 'polish', 'persian', 'dutch', 'danish',
+    # 'french', 'burmese', 'greek', 'italian', 'japanese', 'deutsch', 'kazakh', 'slovak']
+    language: str = 'english'
+
+
+@dataclass
+class EvaluationArguments:
+    testset_file: str
+    critic_llm: Dict = field(default_factory=dict)
+    embeddings: Dict = field(default_factory=dict)
+    metrics: List[str] = field(
+        default_factory=lambda: ['answer_relevancy', 'faithfulness']
+    )
+    # For LLM based evaluation
+    # available: ['english', 'hindi', 'marathi', 'chinese', 'spanish', 'amharic', 'arabic',
+    # 'armenian', 'bulgarian', 'urdu', 'russian', 'polish', 'persian', 'dutch', 'danish',
+    # 'french', 'burmese', 'greek', 'italian', 'japanese', 'deutsch', 'kazakh', 'slovak']
+    language: str = 'english'

evalscope/backend/rag_eval/ragas/metrics/__init__.py

@@ -0,0 +1,2 @@
+from .multi_modal_faithfulness import multimodal_faithness, MultiModalFaithfulness
+from .multi_modal_relevance import multimodal_relevance, MultiModalRelevance

evalscope/backend/rag_eval/ragas/metrics/multi_modal_faithfulness.py

@@ -0,0 +1,91 @@
+import typing as t
+import numpy as np
+from dataclasses import dataclass, field
+from ragas.dataset_schema import SingleTurnSample
+from ragas.metrics.base import MetricWithLLM, SingleTurnMetric, MetricType
+from pydantic import BaseModel, Field
+from evalscope.backend.rag_eval.ragas.prompts.multi_modal_prompt import ImageTextPrompt
+
+
+class FaithfulnessInput(BaseModel):
+    response: str = Field(description="response from AI")
+    retrieved_contexts: list[str] = Field(description="contexts retrieved from the LLM")
+
+    def to_string_list(self):
+        return [
+            "inputs:",
+            self.response,
+            "retrieved_contexts: ",
+        ] + self.retrieved_contexts
+
+
+class FaithfulnessOutput(BaseModel):
+    faithful: bool = Field(description="boolean indicating if request was faithful")
+
+
+class MultiModalFaithfulnessPrompt(
+    ImageTextPrompt[FaithfulnessInput, FaithfulnessOutput]
+):
+    # refer: https://github.com/run-llama/llama_index/blob/main/llama-index-core/llama_index/core/evaluation/multi_modal/faithfulness.py
+    instruction = "Please tell if a given piece of information is supported by the visual as well as textual context information. You need to answer with either True or False. Answer True if any of the image(s) and textual context supports the information"
+    input_model = FaithfulnessInput
+    output_model = FaithfulnessOutput
+    examples = [
+        (
+            FaithfulnessInput(
+                response="Apple pie is generally double-crusted.",
+                retrieved_contexts=[
+                    "An apple pie is a fruit pie in which the principal filling ingredient is apples.",
+                    "Apple pie is often served with whipped cream, ice cream ('apple pie à la mode'), custard or cheddar cheese.",
+                    "It is generally double-crusted, with pastry both above and below the filling; the upper crust may be solid or latticed (woven of crosswise strips).",
+                ],
+            ),
+            FaithfulnessOutput(faithful=True),
+        ),
+        (
+            FaithfulnessInput(
+                response="Apple pies tastes bad.",
+                retrieved_contexts=[
+                    "An apple pie is a fruit pie in which the principal filling ingredient is apples.",
+                    "Apple pie is often served with whipped cream, ice cream ('apple pie à la mode'), custard or cheddar cheese.",
+                    "It is generally double-crusted, with pastry both above and below the filling; the upper crust may be solid or latticed (woven of crosswise strips).",
+                ],
+            ),
+            FaithfulnessOutput(faithful=False),
+        ),
+    ]
+
+
+@dataclass
+class MultiModalFaithfulness(MetricWithLLM, SingleTurnMetric):
+    name: str = "faithful_rate"  # type: ignore
+    _required_columns: t.Dict[MetricType, t.Set[str]] = field(
+        default_factory=lambda: {
+            MetricType.SINGLE_TURN: {
+                "response",
+                "retrieved_contexts",
+            }
+        }
+    )
+    faithfulness_prompt: ImageTextPrompt = MultiModalFaithfulnessPrompt()
+
+    async def _ascore(self, row: t.Dict, callbacks) -> float:
+        prompt_input = FaithfulnessInput(
+            response=row["response"], retrieved_contexts=row["retrieved_contexts"]
+        )
+        assert self.llm is not None, "LLM is not set"
+        prompt_response = await self.faithfulness_prompt.generate(
+            data=prompt_input, llm=self.llm, callbacks=callbacks
+        )
+        if prompt_response is None:
+            return np.nan
+        return float(prompt_response.faithful)
+
+    async def _single_turn_ascore(
+        self, sample: SingleTurnSample, callbacks
+    ) -> float:
+        row = sample.to_dict()
+        return await self._ascore(row, callbacks)
+
+
+multimodal_faithness = MultiModalFaithfulness()

evalscope/backend/rag_eval/ragas/metrics/multi_modal_relevance.py

@@ -0,0 +1,99 @@
+import typing as t
+from dataclasses import dataclass, field
+import numpy as np
+from ragas.dataset_schema import SingleTurnSample
+from ragas.metrics.base import MetricWithLLM, SingleTurnMetric, MetricType
+from pydantic import BaseModel, Field
+from evalscope.backend.rag_eval.ragas.prompts.multi_modal_prompt import ImageTextPrompt
+
+
+class RelevanceInput(BaseModel):
+    user_input: str = Field(description="user input")
+    response: str = Field(description="response from AI")
+    retrieved_contexts: list[str] = Field(description="contexts retrieved from the LLM")
+
+    def to_string_list(self):
+        return [
+            f"Question: {self.user_input}",
+            f"Response: {self.response}",
+            "retrieved_contexts: ",
+        ] + self.retrieved_contexts
+
+
+class RelevanceOutput(BaseModel):
+    relevance: bool = Field(description="boolean indicating if request was relevance")
+
+
+class MultiModalRelevancePrompt(ImageTextPrompt[RelevanceInput, RelevanceOutput]):
+    # refer https://github.com/run-llama/llama_index/blob/main/llama-index-core/llama_index/core/evaluation/multi_modal/relevancy.py
+    instruction = """
+Your task is to evaluate if the response for the query is in line with the images and textual context information provided.
+You have two options to answer. Either True / False.
+Answer - True, if the response for the query is in line with context information otherwise False.
+"""
+    input_model = RelevanceInput
+    output_model = RelevanceOutput
+    examples = [
+        (
+            RelevanceInput(
+                user_input="What is the primary ingredient in a traditional Margherita pizza?",
+                response="The primary ingredients in a Margherita pizza are tomatoes, mozzarella cheese, and fresh basil.",
+                retrieved_contexts=[
+                    "A traditional Margherita pizza consists of a thin crust.",
+                    "The main toppings include tomatoes, mozzarella cheese, fresh basil, salt, and olive oil.",
+                    "It is one of the simplest and most classic types of pizza.",
+                ],
+            ),
+            RelevanceOutput(relevance=True),
+        ),
+        (
+            RelevanceInput(
+                user_input="Who won the Best Actor award at the Oscars in 2021?",
+                response="The Best Actor award in 2021 was won by Leonardo DiCaprio.",
+                retrieved_contexts=[
+                    "The 93rd Academy Awards were held in 2021.",
+                    "Anthony Hopkins won the Best Actor award for his role in 'The Father'.",
+                    "The event was unique due to COVID-19 restrictions.",
+                ],
+            ),
+            RelevanceOutput(relevance=False),
+        ),
+    ]
+
+
+@dataclass
+class MultiModalRelevance(MetricWithLLM, SingleTurnMetric):
+    name: str = "relevance_rate"  # type: ignore
+    _required_columns: t.Dict[MetricType, t.Set[str]] = field(
+        default_factory=lambda: {
+            MetricType.SINGLE_TURN: {
+                "user_input",
+                "response",
+                "retrieved_contexts",
+            }
+        }
+    )
+    relevance_prompt: ImageTextPrompt = MultiModalRelevancePrompt()
+
+    async def _ascore(self, row: t.Dict, callbacks) -> float:
+        prompt_input = RelevanceInput(
+            user_input=row["user_input"],
+            response=row["response"],
+            retrieved_contexts=row["retrieved_contexts"],
+        )
+        assert self.llm is not None, "LLM is not set"
+        prompt_response = await self.relevance_prompt.generate(
+            data=prompt_input, llm=self.llm, callbacks=callbacks
+        )
+        if prompt_response is None:
+            return np.nan
+        return float(prompt_response.relevance)
+
+    async def _single_turn_ascore(
+        self, sample: SingleTurnSample, callbacks
+    ) -> float:
+        row = sample.to_dict()
+        return await self._ascore(row, callbacks)
+
+
+multimodal_relevance = MultiModalRelevance()

evalscope/backend/rag_eval/ragas/task_template.py

@@ -0,0 +1,61 @@
+import os
+import asyncio
+from datasets import Dataset
+from evalscope.backend.rag_eval import EmbeddingModel, LLM
+from evalscope.backend.rag_eval.ragas.tasks.translate_prompt import translate_prompts
+from evalscope.utils.logger import get_logger
+from .arguments import EvaluationArguments
+
+logger = get_logger()
+
+
+def rag_eval(
+    args: EvaluationArguments,
+) -> None:
+
+    from ragas import evaluate, RunConfig
+    from ragas.llms import LangchainLLMWrapper
+    import importlib
+
+    def dynamic_import(*function_names):
+        functions = []
+        for name in function_names:
+            module = importlib.import_module('ragas.metrics')
+            functions.append(getattr(module, name)())
+        return functions
+
+    llm = LLM.load(**args.critic_llm)
+    embedding = EmbeddingModel.load(**args.embeddings)
+
+    # load dataset
+    dataset = Dataset.from_json(args.testset_file)
+
+    # load metrics
+    metrics = dynamic_import(*args.metrics)
+    asyncio.run(
+        translate_prompts(
+            prompts=metrics,
+            target_lang=args.language,
+            llm=LangchainLLMWrapper(llm),
+            adapt_instruction=True,
+        )
+    )
+
+    # evaluate
+    runconfig = RunConfig(timeout=600, max_retries=2, max_wait=60, max_workers=1)
+    score = evaluate(
+        dataset,
+        metrics=metrics,
+        llm=llm,
+        embeddings=embedding,
+        run_config=runconfig,
+    )
+    score_df = score.to_pandas()
+    logger.info(score_df)
+
+    output_path = args.testset_file.replace('.json', '_score.json')
+    score_df.to_json(
+        output_path, indent=4, index=False, orient='records', force_ascii=False
+    )
+
+    logger.info(f'Eval score saved to {output_path}')

evalscope/backend/rag_eval/ragas/tasks/__init__.py

@@ -0,0 +1,2 @@
+from evalscope.backend.rag_eval.ragas.tasks.testset_generation import generate_testset
+from evalscope.backend.rag_eval.ragas.tasks.translate_prompt import translate_prompts