evalscope 0.5.5__py3-none-any.whl → 0.5.5rc0__py3-none-any.whl

evalscope/backend/rag_eval/cmteb/tasks/__init__.py

@@ -1,64 +0,0 @@
-from .Classification import *
-from .Clustering import *
-from .PairClassification import *
-from .Reranking import *
-from .Retrieval import *
-from .STS import *
-
-CLS_CLASSIFICATION = {
-    "TNews": TNews,
-    "IFlyTek": IFlyTek,
-    "MultilingualSentiment": MultilingualSentiment,
-    "JDReview": JDReview,
-    "OnlineShopping": OnlineShopping,
-    "Waimai": Waimai,
-}
-
-CLS_CLUSTERING = {
-    "CLSClusteringS2S": CLSClusteringFastS2S,
-    "CLSClusteringP2P": CLSClusteringFastP2P,
-    "ThuNewsClusteringS2S": ThuNewsClusteringFastS2S,
-    "ThuNewsClusteringP2P": ThuNewsClusteringFastP2P,
-}
-
-CLS_PAIR_CLASSIFICATION = {
-    "Ocnli": Ocnli,
-    "Cmnli": Cmnli,
-}
-
-CLS_RERANKING = {
-    "T2Reranking": T2Reranking,
-    "MMarcoReranking": MMarcoReranking,
-    "CMedQAv1": CMedQAv1,
-    "CMedQAv2": CMedQAv2,
-}
-
-CLS_RETRIEVAL = {
-    "T2Retrieval": T2Retrieval,
-    "MMarcoRetrieval": MMarcoRetrieval,
-    "DuRetrieval": DuRetrieval,
-    "CovidRetrieval": CovidRetrieval,
-    "CmedqaRetrieval": CmedqaRetrieval,
-    "EcomRetrieval": EcomRetrieval,
-    "MedicalRetrieval": MedicalRetrieval,
-    "VideoRetrieval": VideoRetrieval,
-}
-
-CLS_STS = {
-    "ATEC": ATEC,
-    "BQ": BQ,
-    "LCQMC": LCQMC,
-    "PAWSX": PAWSX,
-    "STSB": STSB,
-    "AFQMC": AFQMC,
-    "QBQTC": QBQTC,
-}
-
-CLS_DICT = {
-    **CLS_CLASSIFICATION,
-    **CLS_CLUSTERING,
-    **CLS_PAIR_CLASSIFICATION,
-    **CLS_RERANKING,
-    **CLS_RETRIEVAL,
-    **CLS_STS,
-}

evalscope/backend/rag_eval/ragas/__init__.py

@@ -1,2 +0,0 @@
-from evalscope.backend.rag_eval.ragas.arguments import TestsetGenerationArguments, EvaluationArguments
-from evalscope.backend.rag_eval.ragas.task_template import testset_generation, rag_eval

evalscope/backend/rag_eval/ragas/arguments.py

@@ -1,37 +0,0 @@
-from dataclasses import dataclass, field
-from typing import List, Optional, Union, Dict, Any
-
-
-@dataclass
-class TestsetGenerationArguments:
-    docs: List[str] = field(default_factory=list)
-    test_size: int = 10
-    output_file: str = "outputs/testset.json"
-    """
-    For local LLM support, you can use the following fields:
-        model_name_or_path: str
-        model_revision: str = "master"
-        template_type: str = "default"
-        generation_config: Optional[Dict]
-        
-    For API LLM support, you can use the following fields:
-        model_name="gpt-4o-mini"
-        api_base: str = "",
-        api_key: Optional[str] = None
-    """
-    generator_llm: Dict = field(default_factory=dict)
-    critic_llm: Dict = field(default_factory=dict)
-    embeddings: Dict = field(default_factory=dict)
-    distribution: str = field(
-        default_factory=lambda: {"simple": 0.5, "multi_context": 0.4, "reasoning": 0.1}
-    )
-
-
-@dataclass
-class EvaluationArguments:
-    testset_file: str
-    critic_llm: Dict = field(default_factory=dict)
-    embeddings: Dict = field(default_factory=dict)
-    metrics: List[str] = field(
-        default_factory=lambda: ["answer_relevancy", "faithfulness"]
-    )

evalscope/backend/rag_eval/ragas/task_template.py

@@ -1,117 +0,0 @@
-import os
-from evalscope.backend.rag_eval import EmbeddingModel, LLM
-from evalscope.utils.logger import get_logger
-from .arguments import TestsetGenerationArguments, EvaluationArguments
-
-logger = get_logger()
-
-
-def testset_generation(args: TestsetGenerationArguments) -> None:
-    from langchain_community.document_loaders.unstructured import UnstructuredFileLoader
-    from ragas.testset.generator import TestsetGenerator
-    from ragas.testset.evolutions import simple, reasoning, multi_context
-    from ragas import RunConfig
-
-    # load data
-    file_path = args.docs
-    loader = UnstructuredFileLoader(file_path, mode="elements")
-    data = loader.load()
-
-    # generator with models
-    generator_llm = LLM.load(**args.generator_llm)
-    critic_llm = LLM.load(**args.critic_llm)
-    embeddings = EmbeddingModel.load(**args.embeddings)
-
-    # Change resulting question type distribution
-    distributions = {
-        simple: args.distribution["simple"],
-        multi_context: args.distribution["multi_context"],
-        reasoning: args.distribution["reasoning"],
-    }
-
-    generator = TestsetGenerator.from_langchain(generator_llm, critic_llm, embeddings)
-
-    runconfig = RunConfig(timeout=30, max_retries=1, max_wait=30, max_workers=1)
-    testset = generator.generate_with_langchain_docs(
-        data,
-        args.test_size,
-        distributions,
-        with_debugging_logs=True,
-        is_async=False,
-        run_config=runconfig,
-    )
-
-    # save file
-    testset_df = testset.to_pandas()
-    output_path = os.path.dirname(args.output_file)
-    os.makedirs(output_path, exist_ok=True)
-    testset_df.to_json(args.output_file, indent=4, index=False, orient="records")
-
-    # get answer
-    testset_with_answer = get_answer(testset_df, generator_llm)
-    testset_with_answer.to_json(
-        args.output_file, indent=4, index=False, orient="records"
-    )
-
-
-def get_answer(testset_df, generator_llm):
-    template = """You are an assistant for question-answering tasks. 
-Use the following pieces of retrieved context to answer the question. 
-If you don't know the answer, just say that you don't know. 
-Use two sentences maximum and keep the answer concise.
-Question: {question} 
-Context: {contexts} 
-Answer:
-"""
-    answers = []
-    for index, row in testset_df.iterrows():
-        question = row["question"]
-        contexts = "\n".join(row["contexts"])
-
-        # Combine question and contexts as input for the LLM
-        input_text = template.format(question=question, contexts=contexts)
-
-        # Generate the answer using the generator LLM
-        answer = generator_llm.invoke(input_text)
-        answers.append(answer)
-
-    testset_df["answer"] = answers
-    return testset_df
-
-
-def rag_eval(
-    args: EvaluationArguments,
-) -> None:
-    from datasets import Dataset
-    from ragas import evaluate
-    from evalscope.backend.rag_eval import EmbeddingModel, LLM
-    from ragas import RunConfig
-    import importlib
-
-    def dynamic_import(module_name, *function_names):
-        # 动态导入指定模块
-        module = importlib.import_module(module_name)
-
-        functions = [getattr(module, name) for name in function_names]
-        return functions
-
-    llm = LLM.load(**args.critic_llm)
-    embedding = EmbeddingModel.load(**args.embeddings)
-
-    dataset = Dataset.from_json(args.testset_file)
-
-    runconfig = RunConfig(timeout=30, max_retries=1, max_wait=30, max_workers=1)
-    score = evaluate(
-        dataset,
-        metrics=dynamic_import("ragas.metrics", *args.metrics),
-        llm=llm,
-        embeddings=embedding,
-        run_config=runconfig,
-    )
-    score_df = score.to_pandas()
-    # logger.info(score_df.to_string())
-
-    output_path = args.testset_file.split(".")[0] + "_score.json"
-    score_df.to_json(output_path, indent=4, index=False, orient="records")
-
-    logger.info(f"Eval score saved to {output_path}")