evalscope 0.5.5__py3-none-any.whl → 0.5.5rc1__py3-none-any.whl

evalscope/backend/rag_eval/cmteb/tasks/STS.py

@@ -1,302 +0,0 @@
-from mteb.abstasks.AbsTaskSTS import AbsTaskSTS
-from mteb.abstasks.TaskMetadata import TaskMetadata
-
-class ATEC(AbsTaskSTS):
-    metadata = TaskMetadata(
-        name="ATEC",
-        dataset={
-            "path": "C-MTEB/ATEC",
-            "revision": "0f319b1142f28d00e055a6770f3f726ae9b7d865",
-        },
-        description="A Chinese dataset for textual relatedness",
-        reference="https://aclanthology.org/2021.emnlp-main.357",
-        type="STS",
-        category="s2s",
-        modalities=["text"],
-        eval_splits=["validation", "test"],
-        eval_langs=["cmn-Hans"],
-        main_score="cosine_spearman",
-        date=None,
-        domains=None,
-        task_subtypes=None,
-        license=None,
-        annotations_creators=None,
-        dialect=None,
-        sample_creation=None,
-        bibtex_citation="""@inproceedings{raghu-etal-2021-end,
-    title = "End-to-End Learning of Flowchart Grounded Task-Oriented Dialogs",
-    author = "Raghu, Dinesh  and
-      Agarwal, Shantanu  and
-      Joshi, Sachindra  and
-      {Mausam}",
-    editor = "Moens, Marie-Francine  and
-      Huang, Xuanjing  and
-      Specia, Lucia  and
-      Yih, Scott Wen-tau",
-    booktitle = "Proceedings of the 2021 Conference on Empirical Methods in Natural Language Processing",
-    month = nov,
-    year = "2021",
-    address = "Online and Punta Cana, Dominican Republic",
-    publisher = "Association for Computational Linguistics",
-    url = "https://aclanthology.org/2021.emnlp-main.357",
-    doi = "10.18653/v1/2021.emnlp-main.357",
-    pages = "4348--4366",
-    abstract = "We propose a novel problem within end-to-end learning of task oriented dialogs (TOD), in which the dialog system mimics a troubleshooting agent who helps a user by diagnosing their problem (e.g., car not starting). Such dialogs are grounded in domain-specific flowcharts, which the agent is supposed to follow during the conversation. Our task exposes novel technical challenges for neural TOD, such as grounding an utterance to the flowchart without explicit annotation, referring to additional manual pages when user asks a clarification question, and ability to follow unseen flowcharts at test time. We release a dataset (FLODIAL) consisting of 2,738 dialogs grounded on 12 different troubleshooting flowcharts. We also design a neural model, FLONET, which uses a retrieval-augmented generation architecture to train the dialog agent. Our experiments find that FLONET can do zero-shot transfer to unseen flowcharts, and sets a strong baseline for future research.",
-}""",
-        descriptive_stats={"n_samples": None, "avg_character_length": None},
-    )
-
-    @property
-    def metadata_dict(self) -> dict[str, str]:
-        metadata_dict = super().metadata_dict
-        metadata_dict["min_score"] = 0
-        metadata_dict["max_score"] = 1
-        return metadata_dict
-
-
-class BQ(AbsTaskSTS):
-    metadata = TaskMetadata(
-        name="BQ",
-        dataset={
-            "path": "C-MTEB/BQ",
-            "revision": "e3dda5e115e487b39ec7e618c0c6a29137052a55",
-        },
-        description="A Chinese dataset for textual relatedness",
-        reference="https://aclanthology.org/2021.emnlp-main.357",
-        type="STS",
-        category="s2s",
-        modalities=["text"],
-        eval_splits=["validation", "test"],
-        eval_langs=["cmn-Hans"],
-        main_score="cosine_spearman",
-        date=None,
-        domains=None,
-        task_subtypes=None,
-        license=None,
-        annotations_creators=None,
-        dialect=None,
-        sample_creation=None,
-        bibtex_citation="""@misc{xiao2024cpackpackagedresourcesadvance,
-      title={C-Pack: Packaged Resources To Advance General Chinese Embedding}, 
-      author={Shitao Xiao and Zheng Liu and Peitian Zhang and Niklas Muennighoff and Defu Lian and Jian-Yun Nie},
-      year={2024},
-      eprint={2309.07597},
-      archivePrefix={arXiv},
-      primaryClass={cs.CL},
-      url={https://arxiv.org/abs/2309.07597}, 
-}""",
-        descriptive_stats={"n_samples": None, "avg_character_length": None},
-    )
-
-    @property
-    def metadata_dict(self) -> dict[str, str]:
-        metadata_dict = super().metadata_dict
-        metadata_dict["min_score"] = 0
-        metadata_dict["max_score"] = 1
-        return metadata_dict
-
-
-class LCQMC(AbsTaskSTS):
-    metadata = TaskMetadata(
-        name="LCQMC",
-        dataset={
-            "path": "C-MTEB/LCQMC",
-            "revision": "17f9b096f80380fce5ed12a9be8be7784b337daf",
-        },
-        description="A Chinese dataset for textual relatedness",
-        reference="https://aclanthology.org/2021.emnlp-main.357",
-        type="STS",
-        category="s2s",
-        modalities=["text"],
-        eval_splits=["test"],
-        eval_langs=["cmn-Hans"],
-        main_score="cosine_spearman",
-        date=None,
-        domains=None,
-        task_subtypes=None,
-        license=None,
-        annotations_creators=None,
-        dialect=None,
-        sample_creation=None,
-        bibtex_citation="""@misc{xiao2024cpackpackagedresourcesadvance,
-      title={C-Pack: Packaged Resources To Advance General Chinese Embedding}, 
-      author={Shitao Xiao and Zheng Liu and Peitian Zhang and Niklas Muennighoff and Defu Lian and Jian-Yun Nie},
-      year={2024},
-      eprint={2309.07597},
-      archivePrefix={arXiv},
-      primaryClass={cs.CL},
-      url={https://arxiv.org/abs/2309.07597}, 
-}""",
-        descriptive_stats={"n_samples": None, "avg_character_length": None},
-    )
-
-    @property
-    def metadata_dict(self) -> dict[str, str]:
-        metadata_dict = super().metadata_dict
-        metadata_dict["min_score"] = 0
-        metadata_dict["max_score"] = 1
-        return metadata_dict
-
-
-class PAWSX(AbsTaskSTS):
-    metadata = TaskMetadata(
-        name="PAWSX",
-        dataset={
-            "path": "C-MTEB/PAWSX",
-            "revision": "9c6a90e430ac22b5779fb019a23e820b11a8b5e1",
-        },
-        description="A Chinese dataset for textual relatedness",
-        reference="https://aclanthology.org/2021.emnlp-main.357",
-        type="STS",
-        category="s2s",
-        modalities=["text"],
-        eval_splits=["test"],
-        eval_langs=["cmn-Hans"],
-        main_score="cosine_spearman",
-        date=None,
-        domains=None,
-        task_subtypes=None,
-        license=None,
-        annotations_creators=None,
-        dialect=None,
-        sample_creation=None,
-        bibtex_citation="""@misc{xiao2024cpackpackagedresourcesadvance,
-      title={C-Pack: Packaged Resources To Advance General Chinese Embedding}, 
-      author={Shitao Xiao and Zheng Liu and Peitian Zhang and Niklas Muennighoff and Defu Lian and Jian-Yun Nie},
-      year={2024},
-      eprint={2309.07597},
-      archivePrefix={arXiv},
-      primaryClass={cs.CL},
-      url={https://arxiv.org/abs/2309.07597}, 
-}""",
-        descriptive_stats={"n_samples": None, "avg_character_length": None},
-    )
-
-    @property
-    def metadata_dict(self) -> dict[str, str]:
-        metadata_dict = super().metadata_dict
-        metadata_dict["min_score"] = 0
-        metadata_dict["max_score"] = 1
-        return metadata_dict
-
-
-class STSB(AbsTaskSTS):
-    metadata = TaskMetadata(
-        name="STSB",
-        dataset={
-            "path": "C-MTEB/STSB",
-            "revision": "0cde68302b3541bb8b3c340dc0644b0b745b3dc0",
-        },
-        description="A Chinese dataset for textual relatedness",
-        reference="https://aclanthology.org/2021.emnlp-main.357",
-        type="STS",
-        category="s2s",
-        modalities=["text"],
-        eval_splits=["validation", "test"],
-        eval_langs=["cmn-Hans"],
-        main_score="cosine_spearman",
-        date=None,
-        domains=None,
-        task_subtypes=None,
-        license=None,
-        annotations_creators=None,
-        dialect=None,
-        sample_creation=None,
-        bibtex_citation="""@misc{xiao2024cpackpackagedresourcesadvance,
-      title={C-Pack: Packaged Resources To Advance General Chinese Embedding}, 
-      author={Shitao Xiao and Zheng Liu and Peitian Zhang and Niklas Muennighoff and Defu Lian and Jian-Yun Nie},
-      year={2024},
-      eprint={2309.07597},
-      archivePrefix={arXiv},
-      primaryClass={cs.CL},
-      url={https://arxiv.org/abs/2309.07597}, 
-}""",
-        descriptive_stats={"n_samples": None, "avg_character_length": None},
-    )
-
-    @property
-    def metadata_dict(self) -> dict[str, str]:
-        metadata_dict = super().metadata_dict
-        metadata_dict["min_score"] = 0
-        metadata_dict["max_score"] = 5
-        return metadata_dict
-
-
-class AFQMC(AbsTaskSTS):
-    metadata = TaskMetadata(
-        name="AFQMC",
-        dataset={
-            "path": "C-MTEB/AFQMC",
-            "revision": "b44c3b011063adb25877c13823db83bb193913c4",
-        },
-        description="A Chinese dataset for textual relatedness",
-        reference="https://aclanthology.org/2021.emnlp-main.357",
-        type="STS",
-        category="s2s",
-        modalities=["text"],
-        eval_splits=["validation"],
-        eval_langs=["cmn-Hans"],
-        main_score="cosine_spearman",
-        date=None,
-        domains=None,
-        task_subtypes=None,
-        license=None,
-        annotations_creators=None,
-        dialect=None,
-        sample_creation=None,
-        bibtex_citation="""@inproceedings{raghu-etal-2021-end,
-    title = "End-to-End Learning of Flowchart Grounded Task-Oriented Dialogs",
-    author = "Raghu, Dinesh  and
-      Agarwal, Shantanu  and
-      Joshi, Sachindra  and
-      {Mausam}",
-    editor = "Moens, Marie-Francine  and
-      Huang, Xuanjing  and
-      Specia, Lucia  and
-      Yih, Scott Wen-tau",
-    booktitle = "Proceedings of the 2021 Conference on Empirical Methods in Natural Language Processing",
-    month = nov,
-    year = "2021",
-    address = "Online and Punta Cana, Dominican Republic",
-    publisher = "Association for Computational Linguistics",
-    url = "https://aclanthology.org/2021.emnlp-main.357",
-    doi = "10.18653/v1/2021.emnlp-main.357",
-    pages = "4348--4366",
-    abstract = "We propose a novel problem within end-to-end learning of task oriented dialogs (TOD), in which the dialog system mimics a troubleshooting agent who helps a user by diagnosing their problem (e.g., car not starting). Such dialogs are grounded in domain-specific flowcharts, which the agent is supposed to follow during the conversation. Our task exposes novel technical challenges for neural TOD, such as grounding an utterance to the flowchart without explicit annotation, referring to additional manual pages when user asks a clarification question, and ability to follow unseen flowcharts at test time. We release a dataset (FLODIAL) consisting of 2,738 dialogs grounded on 12 different troubleshooting flowcharts. We also design a neural model, FLONET, which uses a retrieval-augmented generation architecture to train the dialog agent. Our experiments find that FLONET can do zero-shot transfer to unseen flowcharts, and sets a strong baseline for future research.",
-}""",
-        descriptive_stats={"n_samples": None, "avg_character_length": None},
-    )
-
-    @property
-    def metadata_dict(self) -> dict[str, str]:
-        metadata_dict = super().metadata_dict
-        metadata_dict["min_score"] = 0
-        metadata_dict["max_score"] = 1
-        return metadata_dict
-
-
-class QBQTC(AbsTaskSTS):
-    metadata = TaskMetadata(
-        name="QBQTC",
-        dataset={
-            "path": "C-MTEB/QBQTC",
-            "revision": "790b0510dc52b1553e8c49f3d2afb48c0e5c48b7",
-        },
-        description="",
-        reference="https://github.com/CLUEbenchmark/QBQTC/tree/main/dataset",
-        type="STS",
-        category="s2s",
-        modalities=["text"],
-        eval_splits=["test"],
-        eval_langs=["cmn-Hans"],
-        main_score="cosine_spearman",
-        date=None,
-        domains=None,
-        task_subtypes=None,
-        license=None,
-        annotations_creators=None,
-        dialect=None,
-        sample_creation=None,
-        bibtex_citation=None,
-        descriptive_stats={"n_samples": None, "avg_character_length": None},
-    )

evalscope/backend/rag_eval/cmteb/tasks/__init__.py

@@ -1,64 +0,0 @@
-from .Classification import *
-from .Clustering import *
-from .PairClassification import *
-from .Reranking import *
-from .Retrieval import *
-from .STS import *
-
-CLS_CLASSIFICATION = {
-    "TNews": TNews,
-    "IFlyTek": IFlyTek,
-    "MultilingualSentiment": MultilingualSentiment,
-    "JDReview": JDReview,
-    "OnlineShopping": OnlineShopping,
-    "Waimai": Waimai,
-}
-
-CLS_CLUSTERING = {
-    "CLSClusteringS2S": CLSClusteringFastS2S,
-    "CLSClusteringP2P": CLSClusteringFastP2P,
-    "ThuNewsClusteringS2S": ThuNewsClusteringFastS2S,
-    "ThuNewsClusteringP2P": ThuNewsClusteringFastP2P,
-}
-
-CLS_PAIR_CLASSIFICATION = {
-    "Ocnli": Ocnli,
-    "Cmnli": Cmnli,
-}
-
-CLS_RERANKING = {
-    "T2Reranking": T2Reranking,
-    "MMarcoReranking": MMarcoReranking,
-    "CMedQAv1": CMedQAv1,
-    "CMedQAv2": CMedQAv2,
-}
-
-CLS_RETRIEVAL = {
-    "T2Retrieval": T2Retrieval,
-    "MMarcoRetrieval": MMarcoRetrieval,
-    "DuRetrieval": DuRetrieval,
-    "CovidRetrieval": CovidRetrieval,
-    "CmedqaRetrieval": CmedqaRetrieval,
-    "EcomRetrieval": EcomRetrieval,
-    "MedicalRetrieval": MedicalRetrieval,
-    "VideoRetrieval": VideoRetrieval,
-}
-
-CLS_STS = {
-    "ATEC": ATEC,
-    "BQ": BQ,
-    "LCQMC": LCQMC,
-    "PAWSX": PAWSX,
-    "STSB": STSB,
-    "AFQMC": AFQMC,
-    "QBQTC": QBQTC,
-}
-
-CLS_DICT = {
-    **CLS_CLASSIFICATION,
-    **CLS_CLUSTERING,
-    **CLS_PAIR_CLASSIFICATION,
-    **CLS_RERANKING,
-    **CLS_RETRIEVAL,
-    **CLS_STS,
-}

evalscope/backend/rag_eval/ragas/__init__.py

@@ -1,2 +0,0 @@
-from evalscope.backend.rag_eval.ragas.arguments import TestsetGenerationArguments, EvaluationArguments
-from evalscope.backend.rag_eval.ragas.task_template import testset_generation, rag_eval

evalscope/backend/rag_eval/ragas/arguments.py

@@ -1,37 +0,0 @@
-from dataclasses import dataclass, field
-from typing import List, Optional, Union, Dict, Any
-
-
-@dataclass
-class TestsetGenerationArguments:
-    docs: List[str] = field(default_factory=list)
-    test_size: int = 10
-    output_file: str = "outputs/testset.json"
-    """
-    For local LLM support, you can use the following fields:
-        model_name_or_path: str
-        model_revision: str = "master"
-        template_type: str = "default"
-        generation_config: Optional[Dict]
-        
-    For API LLM support, you can use the following fields:
-        model_name="gpt-4o-mini"
-        api_base: str = "",
-        api_key: Optional[str] = None
-    """
-    generator_llm: Dict = field(default_factory=dict)
-    critic_llm: Dict = field(default_factory=dict)
-    embeddings: Dict = field(default_factory=dict)
-    distribution: str = field(
-        default_factory=lambda: {"simple": 0.5, "multi_context": 0.4, "reasoning": 0.1}
-    )
-
-
-@dataclass
-class EvaluationArguments:
-    testset_file: str
-    critic_llm: Dict = field(default_factory=dict)
-    embeddings: Dict = field(default_factory=dict)
-    metrics: List[str] = field(
-        default_factory=lambda: ["answer_relevancy", "faithfulness"]
-    )

evalscope/backend/rag_eval/ragas/task_template.py

@@ -1,117 +0,0 @@
-import os
-from evalscope.backend.rag_eval import EmbeddingModel, LLM
-from evalscope.utils.logger import get_logger
-from .arguments import TestsetGenerationArguments, EvaluationArguments
-
-logger = get_logger()
-
-
-def testset_generation(args: TestsetGenerationArguments) -> None:
-    from langchain_community.document_loaders.unstructured import UnstructuredFileLoader
-    from ragas.testset.generator import TestsetGenerator
-    from ragas.testset.evolutions import simple, reasoning, multi_context
-    from ragas import RunConfig
-
-    # load data
-    file_path = args.docs
-    loader = UnstructuredFileLoader(file_path, mode="elements")
-    data = loader.load()
-
-    # generator with models
-    generator_llm = LLM.load(**args.generator_llm)
-    critic_llm = LLM.load(**args.critic_llm)
-    embeddings = EmbeddingModel.load(**args.embeddings)
-
-    # Change resulting question type distribution
-    distributions = {
-        simple: args.distribution["simple"],
-        multi_context: args.distribution["multi_context"],
-        reasoning: args.distribution["reasoning"],
-    }
-
-    generator = TestsetGenerator.from_langchain(generator_llm, critic_llm, embeddings)
-
-    runconfig = RunConfig(timeout=30, max_retries=1, max_wait=30, max_workers=1)
-    testset = generator.generate_with_langchain_docs(
-        data,
-        args.test_size,
-        distributions,
-        with_debugging_logs=True,
-        is_async=False,
-        run_config=runconfig,
-    )
-
-    # save file
-    testset_df = testset.to_pandas()
-    output_path = os.path.dirname(args.output_file)
-    os.makedirs(output_path, exist_ok=True)
-    testset_df.to_json(args.output_file, indent=4, index=False, orient="records")
-
-    # get answer
-    testset_with_answer = get_answer(testset_df, generator_llm)
-    testset_with_answer.to_json(
-        args.output_file, indent=4, index=False, orient="records"
-    )
-
-
-def get_answer(testset_df, generator_llm):
-    template = """You are an assistant for question-answering tasks. 
-Use the following pieces of retrieved context to answer the question. 
-If you don't know the answer, just say that you don't know. 
-Use two sentences maximum and keep the answer concise.
-Question: {question} 
-Context: {contexts} 
-Answer:
-"""
-    answers = []
-    for index, row in testset_df.iterrows():
-        question = row["question"]
-        contexts = "\n".join(row["contexts"])
-
-        # Combine question and contexts as input for the LLM
-        input_text = template.format(question=question, contexts=contexts)
-
-        # Generate the answer using the generator LLM
-        answer = generator_llm.invoke(input_text)
-        answers.append(answer)
-
-    testset_df["answer"] = answers
-    return testset_df
-
-
-def rag_eval(
-    args: EvaluationArguments,
-) -> None:
-    from datasets import Dataset
-    from ragas import evaluate
-    from evalscope.backend.rag_eval import EmbeddingModel, LLM
-    from ragas import RunConfig
-    import importlib
-
-    def dynamic_import(module_name, *function_names):
-        # 动态导入指定模块
-        module = importlib.import_module(module_name)
-
-        functions = [getattr(module, name) for name in function_names]
-        return functions
-
-    llm = LLM.load(**args.critic_llm)
-    embedding = EmbeddingModel.load(**args.embeddings)
-
-    dataset = Dataset.from_json(args.testset_file)
-
-    runconfig = RunConfig(timeout=30, max_retries=1, max_wait=30, max_workers=1)
-    score = evaluate(
-        dataset,
-        metrics=dynamic_import("ragas.metrics", *args.metrics),
-        llm=llm,
-        embeddings=embedding,
-        run_config=runconfig,
-    )
-    score_df = score.to_pandas()
-    # logger.info(score_df.to_string())
-
-    output_path = args.testset_file.split(".")[0] + "_score.json"
-    score_df.to_json(output_path, indent=4, index=False, orient="records")
-
-    logger.info(f"Eval score saved to {output_path}")