graphrag-eval 6.3.0__tar.gz → 6.4.0__tar.gz

{graphrag_eval-6.3.0 → graphrag_eval-6.4.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.3
 Name: graphrag-eval
-Version: 6.3.0
+Version: 6.4.0
 Summary: For assessing question answering systems' final answers and intermediate steps, against a given set of questions, reference answers and steps.
 License: Apache-2.0
 Author: Philip Ganchev

{graphrag_eval-6.3.0 → graphrag_eval-6.4.0}/graphrag_eval/aggregation.py

@@ -1,13 +1,13 @@
 import json
-import yaml
 from collections import defaultdict
 from collections.abc import Sequence
 from pathlib import Path
 from statistics import mean, median
 from typing import Any, Collection, Iterable
 
-from . import evaluation
+import yaml
 
+from . import evaluation
 
 METRICS = [
     "answer_recall",
@@ -155,7 +155,7 @@ def compute_micro_stats(
 ) -> dict:
     if custom_metrics is None:
         custom_metrics = []
-    
+
     values = number_of_samples_per_template_by_status.values()
     micro_summary = defaultdict(dict, {
         "number_of_error_samples": sum(v["error"] for v in values),
@@ -197,7 +197,7 @@ def compute_macro_stats(
 ) -> dict:
     if custom_metrics is None:
         custom_metrics = []
-    
+
     macro_summary = defaultdict(dict)
     for metric in METRICS + custom_metrics:
         means = [

{graphrag_eval-6.3.0 → graphrag_eval-6.4.0}/graphrag_eval/answer_correctness.py

@@ -1,21 +1,36 @@
+from __future__ import annotations
+
 from pathlib import Path
+from typing import Any, Self, TYPE_CHECKING
 
 from pydantic import BaseModel, Field
 
 from graphrag_eval.util import compute_f1
+from .evaluator import Evaluator
+
+if TYPE_CHECKING:
+    from ragas.llms.base import InstructorBaseRagasLLM
 
 
 def load_default_prompt() -> str:
-    with open(Path(__file__).parent / "prompts" / "template.md", "r", encoding="utf-8") as f:
+    with open(
+        Path(__file__).parent / "prompts" / "template.md",
+        encoding="utf-8"
+    ) as f:
         return f.read()
 
 
 class AnswerCorrectnessConfig(BaseModel):
+    enabled: bool = Field(default=True)
     prompt: str = Field(default_factory=load_default_prompt)
 
 
 class InvalidPromptException(Exception):
-    def __init__(self, message="The prompt template is invalid and cannot be formatted."):
+    def __init__(
+        self,
+        message="The prompt template is invalid and cannot be "
+                "formatted."
+    ):
         self.message = message
         super().__init__(self.message)
 
@@ -23,13 +38,25 @@ class InvalidPromptException(Exception):
 class AnswerCorrectnessEvaluator:
     def __init__(
         self,
-        llm: "InstructorBaseRagasLLM",
+        ragas_llm: InstructorBaseRagasLLM,
         config: AnswerCorrectnessConfig | None = None,
     ):
         self.config = config or AnswerCorrectnessConfig()
         self.__validate_prompt_template(self.config.prompt)
         self.prompt_template = self.config.prompt
-        self.llm = llm
+        self.ragas_llm = ragas_llm
+
+    @classmethod
+    def from_config(
+        cls,
+        ragas_llm: InstructorBaseRagasLLM | None,
+        config: AnswerCorrectnessConfig | None
+    ) -> Self | None:
+        if ragas_llm is None:
+            return None
+        if config is None or not config.enabled:
+            return None
+        return cls(ragas_llm=ragas_llm, config=config)
 
     @staticmethod
     def __validate_prompt_template(prompt_template: str):
@@ -48,7 +75,7 @@ class AnswerCorrectnessEvaluator:
 
     async def _agenerate(self, prompt):
         """Wrapper method for easier testing"""
-        return (await self.llm.agenerate(prompt, None)).choices[0].message.content
+        return (await self.ragas_llm.agenerate(prompt, None)).choices[0].message.content
 
     async def evaluate_answer(
         self,
@@ -56,9 +83,13 @@ class AnswerCorrectnessEvaluator:
         reference_answer: str,
         actual_answer: str
     ) -> tuple[int, int, int, str]:
-        if any(not s.strip() for s in [question, reference_answer, actual_answer]):
-            raise ValueError("The question of the reference or the actual answer is a blank "
-                             "string!")
+        if any(
+            not s.strip() for s in [question, reference_answer, actual_answer]
+        ):
+            raise ValueError(
+                "The question of the reference or the actual answer is a blank "
+                "string!"
+            )
         prompt = self.prompt_template.format(
             question=question,
             reference_answer=reference_answer,
@@ -67,12 +98,14 @@ class AnswerCorrectnessEvaluator:
         response_str = await self._agenerate(prompt)
         return self.extract_response_values(response_str)
 
-    async def get_correctness_dict(
+    async def evaluate(
         self,
-        reference: dict,
-        actual: dict,
-    ):
-        result = {"reference_answer": reference["reference_answer"]}
+        reference: dict[str, Any],
+        actual: dict[str, Any]
+    ) -> dict[str, Any]:
+        if "actual_answer" not in actual or "reference_answer" not in reference:
+            return {}
+        result = {}
         try:
             num_ref_claims, num_actual_claims, num_matching_claims, reason = \
                 await self.evaluate_answer(
@@ -96,7 +129,7 @@ class AnswerCorrectnessEvaluator:
             if f1 is not None:
                 result["answer_f1"] = f1
         except Exception as exc:
-            result["answer_eval_error"] = str(exc)
+            result["answer_correctness_error"] = str(exc)
         return result
 
     @staticmethod
@@ -134,6 +167,10 @@ class AnswerCorrectnessEvaluator:
             n_matching > n_actual
         ]):
             raise ValueError(
-                f"Invalid claims counts combination: {n_ref}\t{n_actual}\t{n_matching}"
+                "Invalid claims counts combination: "
+                f"{n_ref}\t{n_actual}\t{n_matching}"
             )
         return n_ref, n_actual, n_matching, vals[3]
+
+
+_: Evaluator = AnswerCorrectnessEvaluator

graphrag_eval-6.4.0/graphrag_eval/answer_relevance.py

@@ -0,0 +1,61 @@
+from __future__ import annotations
+
+from typing import Any, Self, TYPE_CHECKING
+
+from pydantic import BaseModel, Field
+
+from .evaluator import Evaluator
+
+if TYPE_CHECKING:
+    from ragas.llms.base import InstructorBaseRagasLLM
+    from ragas.embeddings.base import BaseRagasEmbeddings, BaseRagasEmbedding
+
+
+class AnswerRelevanceConfig(BaseModel):
+    enabled: bool = Field(default=True)
+
+
+class AnswerRelevanceEvaluator:
+    def __init__(
+        self,
+        ragas_llm: InstructorBaseRagasLLM,
+        ragas_embedder: BaseRagasEmbeddings | BaseRagasEmbedding
+    ):
+        from ragas.metrics.collections import AnswerRelevancy
+        self.scorer = AnswerRelevancy(llm=ragas_llm, embeddings=ragas_embedder)
+
+    @classmethod
+    def from_config(
+        cls,
+        ragas_llm: InstructorBaseRagasLLM | None,
+        ragas_embedder: BaseRagasEmbeddings | BaseRagasEmbedding | None,
+        config: AnswerRelevanceConfig | None
+    ) -> Self | None:
+        if ragas_llm is None or ragas_embedder is None:
+            return None
+        if config is None or not config.enabled:
+            return None
+        return cls(ragas_llm=ragas_llm, ragas_embedder=ragas_embedder)
+
+    async def evaluate(
+        self,
+        reference: dict[str, Any],
+        actual: dict[str, Any]
+    ) -> dict[str, Any]:
+        if "actual_answer" not in actual:
+            return {}
+        try:
+            result = await self.scorer.ascore(
+                user_input=reference["question_text"],
+                response=actual["actual_answer"]
+            )
+            return {
+                "answer_relevance": result.value
+            }
+        except Exception as e:
+            return {
+                "answer_relevance_error": str(e)
+            }
+
+
+_: Evaluator = AnswerRelevanceEvaluator

{graphrag_eval-6.3.0 → graphrag_eval-6.4.0}/graphrag_eval/cli/answer_correctness.py

@@ -1,8 +1,11 @@
+from __future__ import annotations
+
 import argparse
 import asyncio
 import csv
 from argparse import ArgumentParser
 from pathlib import Path
+from typing import TYPE_CHECKING
 
 from tqdm import tqdm
 
@@ -10,34 +13,39 @@ from graphrag_eval import llm_factory
 from graphrag_eval.answer_correctness import AnswerCorrectnessEvaluator
 from graphrag_eval.evaluation import Config
 
+if TYPE_CHECKING:
+    from ragas.llms.base import InstructorBaseRagasLLM
+
 
 def parse_args() -> argparse.Namespace:
     parser = ArgumentParser(
-        description="Calculates answer correctness over the entries from the input tsv file and "
-                    "stores the output in the output tsv file.",
+        description="Calculates answer correctness over the entries from the "
+                    "input tsv file and stores the output in the output tsv "
+                    "file.",
     )
     parser.add_argument(
         "-i",
         "--input-tsv-file-path",
         type=Path,
         required=True,
-        help="Input tsv file path with columns `Question`, `Reference answer` and `Actual answer`",
+        help="Input tsv file path with columns `Question`, `Reference answer` "
+             "and `Actual answer`",
     )
     parser.add_argument(
         "-o",
         "--output-tsv-file-path",
         type=Path,
         required=True,
-        help="Output tsv file path with columns `#Reference`, `#PTarget`, `#Matching`, "
-             "`Reasoning`, `Error`",
+        help="Output tsv file path with columns `#Reference`, `#PTarget`, "
+             "`#Matching`, `Reasoning`, `Error`",
     )
     parser.add_argument(
         "-c",
         "--config-yaml-file-path",
         type=Path,
         required=True,
-        help="Config yaml file path with definition of the LLM to use and optionally a custom "
-             "prompt.",
+        help="Config yaml file path with definition of the LLM to use and "
+             "optionally a custom prompt.",
     )
     return parser.parse_args()
 
@@ -54,7 +62,9 @@ async def evaluate_and_write(
     output_tsv_file_path.parent.mkdir(parents=True, exist_ok=True)
     with open(output_tsv_file_path, "w", encoding="utf-8") as f:
         writer = csv.writer(f, delimiter="\t")
-        writer.writerow(["#Reference", "#PTarget", "#Matching", "Reasoning", "Error"])
+        writer.writerow(
+            ["#Reference", "#PTarget", "#Matching", "Reasoning", "Error"]
+        )
 
         for row in tqdm(rows):
             if "Question" not in row or \
@@ -81,19 +91,26 @@ def run(
     output_tsv_file_path: Path,
 ):
     config = Config.parse(config_yaml_file_path)
-    ragas_llm = llm_factory.create_llm(config)
+    ragas_llm: InstructorBaseRagasLLM | None = llm_factory.create_llm(
+        config.llm
+    )
     if ragas_llm is None:
-        raise ValueError("LLM must be configured to calculate the answer correctness!")
-    else:
-        evaluator = AnswerCorrectnessEvaluator(
-            llm=ragas_llm,
-            config=config.answer_correctness,
+        raise ValueError(
+            "LLM must be configured to calculate the answer correctness!"
+        )
+    if config.answer_correctness and not config.answer_correctness.enabled:
+        raise ValueError(
+            "Can't disable answer correctness, when running this script!"
         )
-        asyncio.run(evaluate_and_write(
-            input_tsv_file_path,
-            output_tsv_file_path,
-            evaluator,
-        ))
+    evaluator = AnswerCorrectnessEvaluator(
+        ragas_llm=ragas_llm,
+        config=config.answer_correctness,
+    )
+    asyncio.run(evaluate_and_write(
+        input_tsv_file_path,
+        output_tsv_file_path,
+        evaluator,
+    ))
 
 
 def main():

{graphrag_eval-6.3.0 → graphrag_eval-6.4.0}/graphrag_eval/custom_evaluation.py

@@ -1,9 +1,14 @@
+from __future__ import annotations
+
 import json
-from typing import Literal
+from typing import Literal, Self, TYPE_CHECKING, Any
 
 from pydantic import BaseModel, ConfigDict, Field, model_validator
 
-from graphrag_eval.llm_factory import create_llm
+from .evaluator import Evaluator
+
+if TYPE_CHECKING:
+    from ragas.llms.base import InstructorBaseRagasLLM
 
 RESERVED_KEYS = {
     "template_id",
@@ -43,7 +48,7 @@ Inputs = Literal[
 StepsKey = Literal["args", "output"]
 
 
-class Config(BaseModel):
+class EvaluatorConfig(BaseModel):
     model_config = ConfigDict(extra='forbid')
     name: str
     inputs: list[Inputs] = Field(..., min_length=1)
@@ -53,7 +58,7 @@ class Config(BaseModel):
     steps_keys: set[StepsKey] | None = Field(default=None, min_length=1)
 
     @model_validator(mode='after')
-    def validate_step_dependencies(self) -> 'Config':
+    def validate_step_dependencies(self) -> Self:
         if set(self.inputs) & {"reference_steps", "actual_steps"}:
             suffix = "is required when steps are in inputs"
             for var_name in ["steps_name", "steps_keys"]:
@@ -62,7 +67,7 @@ class Config(BaseModel):
         return self
 
     @model_validator(mode='after')
-    def validate_name_and_outputs(self) -> 'Config':
+    def validate_name_and_outputs(self) -> Self:
         if self.name + "_error" in RESERVED_KEYS:
             raise ValueError(f"Name {self.name} is reserved")
         conflicting_keys = set(self.outputs.keys()) & RESERVED_KEYS
@@ -76,7 +81,7 @@ def create_input_template(input_key: str) -> str:
     return f"# {header}\n{{{input_key}}}"
 
 
-def create_prompt_template(config: Config, output_variables: list[str]) -> str:
+def create_prompt_template(config: EvaluatorConfig, output_variables: list[str]) -> str:
     """
     Return a template for the LLM prompt, with placeholders for the inputs, 
     instructions, outputs etc. We use this template at evaluation time to
@@ -99,8 +104,8 @@ def create_prompt_template(config: Config, output_variables: list[str]) -> str:
 class CustomEvaluator:
     def __init__(
         self,
-        config: Config,
-        eval_config: "evaluation.Config",
+        ragas_llm: InstructorBaseRagasLLM,
+        config: EvaluatorConfig,
     ):
         self.name = config.name
         self.input_variables = config.inputs
@@ -111,11 +116,24 @@ class CustomEvaluator:
             config,
             self.output_variables
         )
-        self.llm = create_llm(eval_config)
+        self.ragas_llm = ragas_llm
+
+    @classmethod
+    def from_config(
+        cls,
+        ragas_llm: InstructorBaseRagasLLM | None,
+        evaluation_configs: list[EvaluatorConfig] | None
+    ) -> list[Self]:
+        if ragas_llm and evaluation_configs:
+            return [
+                cls(ragas_llm, evaluation_config)
+                for evaluation_config in evaluation_configs
+            ]
+        return []
 
     async def _agenerate(self, prompt: str) -> str:
         """Wrapper method for easier testing"""
-        return (await self.llm.agenerate(prompt, None)).choices[0].message.content
+        return (await self.ragas_llm.agenerate(prompt, None)).choices[0].message.content
 
     def format_steps(self, steps: list) -> str:
         steps_formatted = []
@@ -157,7 +175,11 @@ class CustomEvaluator:
             return result
         return self.error(f"Expected {n_exp} tab-separated values, got: {response}")
 
-    async def evaluate(self, reference: dict, actual: dict) -> dict[str, str | None]:
+    async def evaluate(
+        self,
+        reference: dict[str, Any],
+        actual: dict[str, Any]
+    ) -> dict[str, Any]:
         inputs = {}
         if "question" in self.input_variables:
             if "question_text" not in reference:
@@ -195,10 +217,4 @@ class CustomEvaluator:
         return self.parse_outputs(response)
 
 
-def create_evaluators(config: "evaluation.Config") -> list[CustomEvaluator]:
-    if config.custom_evaluations and config.llm:
-        return [
-            CustomEvaluator(custom_evaluation_config, config)
-            for custom_evaluation_config in config.custom_evaluations
-        ]
-    return []
+_: Evaluator = CustomEvaluator

graphrag_eval-6.4.0/graphrag_eval/evaluation.py

@@ -0,0 +1,154 @@
+from __future__ import annotations
+
+from pathlib import Path
+from typing import Self, TYPE_CHECKING
+
+import yaml
+from pydantic import BaseModel, Field, model_validator
+
+from .answer_correctness import (
+    AnswerCorrectnessConfig,
+    AnswerCorrectnessEvaluator,
+)
+from .answer_relevance import AnswerRelevanceConfig, AnswerRelevanceEvaluator
+from .custom_evaluation import EvaluatorConfig, CustomEvaluator
+from .evaluator import Evaluator
+from .llm_factory import LLMConfig, create_llm, create_embedder
+from .steps.evaluation import evaluate_steps
+
+if TYPE_CHECKING:
+    from ragas.llms.base import InstructorBaseRagasLLM
+    from ragas.embeddings.base import BaseRagasEmbeddings, BaseRagasEmbedding
+
+
+class Config(BaseModel):
+    llm: LLMConfig | None = None
+    custom_evaluations: list[EvaluatorConfig] | None = Field(
+        default=None,
+        min_length=1
+    )
+    answer_correctness: AnswerCorrectnessConfig | None = None
+    answer_relevance: AnswerRelevanceConfig | None = None
+
+    @model_validator(mode="after")
+    def validate_config_and_set_defaults(self) -> Self:
+        has_llm = self.llm is not None
+        has_embedding = has_llm and self.llm.embedding is not None
+
+        if self.answer_correctness is None and has_llm:
+            self.answer_correctness = AnswerCorrectnessConfig()
+
+        if self.answer_relevance is None and has_embedding:
+            self.answer_relevance = AnswerRelevanceConfig()
+
+        if self.custom_evaluations and not has_llm:
+            raise ValueError(
+                "llm config is required if custom_evaluations are provided"
+            )
+        if (
+            self.answer_correctness
+            and self.answer_correctness.enabled
+            and not has_llm
+        ):
+            raise ValueError(
+                "llm config is required if answer correctness is enabled"
+            )
+        if (
+            self.answer_relevance
+            and self.answer_relevance.enabled
+            and not has_embedding
+        ):
+            raise ValueError(
+                "llm config including embedding is required if answer "
+                "relevance is enabled"
+            )
+        return self
+
+    @classmethod
+    def parse(cls, config_file_path: str | Path | None) -> Self:
+        if config_file_path:
+            with open(config_file_path, encoding="utf-8") as f:
+                config_dict = yaml.safe_load(f)
+            return cls(**config_dict)
+        return cls()
+
+
+async def run_evaluation(
+    qa_dataset: list[dict],
+    responses_dict: dict,
+    config_file_path: str | Path | None = None,
+) -> list[dict]:
+    evaluators, ragas_llm = parse_config_and_init_evaluators(config_file_path)
+
+    # Output metrics are not nested, for simpler aggregation
+    evaluation_results = []
+    for template in qa_dataset:
+        template_id = template["template_id"]
+        for question in template["questions"]:
+            actual_result = responses_dict[question["id"]]
+            eval_result = {
+                "template_id": template_id,
+                "question_id": actual_result["question_id"],
+                "question_text": question["question_text"]
+            }
+            for key in ("input_tokens", "output_tokens", "total_tokens",
+                        "elapsed_sec"):
+                if key in actual_result:
+                    eval_result[key] = actual_result[key]
+            if "actual_answer" in actual_result:
+                eval_result["actual_answer"] = actual_result["actual_answer"]
+            if "reference_answer" in question:
+                eval_result["reference_answer"] = question["reference_answer"]
+            if "reference_steps" in question:
+                eval_result["reference_steps"] = question["reference_steps"]
+            if "error" in actual_result:
+                eval_result.update({
+                    "status": "error",
+                    "error": actual_result["error"],
+                })
+            else:
+                eval_result["status"] = "success"
+
+            eval_result.update(
+                await evaluate_steps(question, actual_result, ragas_llm)
+            )
+            for evaluator in evaluators:
+                eval_result.update(
+                    await evaluator.evaluate(question, actual_result)
+                )
+
+            evaluation_results.append(eval_result)
+    return evaluation_results
+
+
+def parse_config_and_init_evaluators(
+    config_file_path: str | Path | None
+) -> tuple[
+    list[Evaluator],
+    InstructorBaseRagasLLM | None,
+]:
+    config = Config.parse(config_file_path)
+    ragas_llm: InstructorBaseRagasLLM | None = create_llm(config.llm)
+    ragas_embedder: BaseRagasEmbeddings | BaseRagasEmbedding | None = (
+        create_embedder(config.llm)
+    )
+
+    evaluators: list[Evaluator] = []
+
+    answer_relevance_evaluator = AnswerRelevanceEvaluator.from_config(
+        ragas_llm, ragas_embedder, config.answer_relevance
+    )
+    if answer_relevance_evaluator:
+        evaluators.append(answer_relevance_evaluator)
+
+    answer_correctness_evaluator = AnswerCorrectnessEvaluator.from_config(
+        ragas_llm, config.answer_correctness
+    )
+    if answer_correctness_evaluator:
+        evaluators.append(answer_correctness_evaluator)
+
+    evaluators.extend(
+        CustomEvaluator.from_config(ragas_llm, config.custom_evaluations)
+    )
+
+    return evaluators, ragas_llm

graphrag_eval-6.4.0/graphrag_eval/evaluator.py

@@ -0,0 +1,14 @@
+from typing import Protocol, Any
+
+
+class Evaluator(Protocol):
+    async def evaluate(
+        self,
+        reference: dict[str, Any],
+        actual: dict[str, Any]
+    ) -> dict[str, Any]:
+        """
+        Evaluate the actual output against the reference.
+        Returns a flat dictionary containing scores or error tracking logs.
+        """
+        ...

{graphrag_eval-6.3.0 → graphrag_eval-6.4.0}/graphrag_eval/llm_factory.py

@@ -1,7 +1,13 @@
-from typing import Optional
+from __future__ import annotations
+
+from typing import TYPE_CHECKING
 
 from pydantic import BaseModel, ConfigDict, Field
 
+if TYPE_CHECKING:
+    from ragas.llms.base import InstructorBaseRagasLLM
+    from ragas.embeddings.base import BaseRagasEmbeddings, BaseRagasEmbedding
+
 
 class GenerationConfig(BaseModel):
     provider: str
@@ -17,18 +23,20 @@ class EmbeddingConfig(BaseModel):
     model_config = ConfigDict(extra='allow')
 
 
-class Config(BaseModel):
+class LLMConfig(BaseModel):
     generation: GenerationConfig
     embedding: EmbeddingConfig | None = None
 
 
-def create_llm(config: "evaluation.Config") -> Optional["InstructorBaseRagasLLM"]:
-    if config.llm:
+def create_llm(
+    config: LLMConfig | None
+) -> InstructorBaseRagasLLM | None:
+    if config:
         import litellm
         from ragas.llms import llm_factory
 
         litellm.drop_params = True  # Remove unsupported params from requests
-        params = config.llm.generation.model_dump()
+        params = config.generation.model_dump()
         ragas_llm = llm_factory(
             provider="litellm",
             model=f"{params.pop('provider')}/{params.pop('model')}",
@@ -40,13 +48,15 @@ def create_llm(config: "evaluation.Config") -> Optional["InstructorBaseRagasLLM"
     return None
 
 
-def create_embedder(config: "evaluation.Config") -> Optional["BaseRagasEmbedding"]:
-    if config.llm and config.llm.embedding:
+def create_embedder(
+    config: LLMConfig | None
+) -> BaseRagasEmbeddings | BaseRagasEmbedding | None:
+    if config and config.embedding:
         import litellm
         from ragas.embeddings.base import embedding_factory
 
         litellm.drop_params = True  # Remove unsupported params from requests
-        params = config.llm.embedding.model_dump()
+        params = config.embedding.model_dump()
         ragas_embedder = embedding_factory(
             provider="litellm",
             model=f"{params.pop('provider')}/{params.pop('model')}",

{graphrag_eval-6.3.0 → graphrag_eval-6.4.0}/graphrag_eval/steps/evaluation.py

@@ -1,13 +1,21 @@
+from __future__ import annotations
+
 import json
 import logging
 from collections import defaultdict
 from collections.abc import Sequence
-from typing import Any
+from typing import Any, TYPE_CHECKING
 
 from .iri_discovery import do_iri_discovery_steps_equal
 from .retrieval_context_ids import recall_at_k
 from .sparql import compare_sparql_results
-from .timeseries import do_retrieve_time_series_steps_equal, do_retrieve_data_points_steps_equal
+from .timeseries import (
+    do_retrieve_time_series_steps_equal,
+    do_retrieve_data_points_steps_equal,
+)
+
+if TYPE_CHECKING:
+    from ragas.llms.base import InstructorBaseRagasLLM
 
 logger = logging.getLogger(__name__)
 
@@ -140,7 +148,7 @@ def calculate_steps_score(
 async def evaluate_steps(
     reference: dict,
     actual: dict,
-    ragas_llm: "InstructorBaseRagasLLM",
+    ragas_llm: InstructorBaseRagasLLM | None,
 ) -> dict:
     eval_result = {}
     actual_steps = actual.get("actual_steps", [])

{graphrag_eval-6.3.0 → graphrag_eval-6.4.0}/pyproject.toml

@@ -1,6 +1,6 @@
 [project]
 name = "graphrag-eval"
-version = "6.3.0"
+version = "6.4.0"
 description = "For assessing question answering systems' final answers and intermediate steps, against a given set of questions, reference answers and steps."
 authors = [
     { name = "Philip Ganchev", email = "philip.ganchev@graphwise.ai" },

graphrag_eval-6.3.0/graphrag_eval/answer_relevance.py

@@ -1,29 +0,0 @@
-from ragas.embeddings.base import BaseRagasEmbedding
-from ragas.llms.base import InstructorBaseRagasLLM
-from ragas.metrics.collections import AnswerRelevancy
-
-from graphrag_eval.util import singleton
-
-
-@singleton
-class Evaluator:
-    def __init__(self, ragas_llm: InstructorBaseRagasLLM, ragas_embedder: BaseRagasEmbedding):
-        self.scorer = AnswerRelevancy(llm=ragas_llm, embeddings=ragas_embedder)
-
-    async def get_relevance_dict(
-        self,
-        question_text: str,
-        actual_answer: str,
-    ) -> dict:
-        try:
-            result = await self.scorer.ascore(
-                user_input=question_text,
-                response=actual_answer
-            )
-            return {
-                "answer_relevance": result.value
-            }
-        except Exception as e:
-            return {
-                "answer_relevance_error": str(e)
-            }

graphrag_eval-6.3.0/graphrag_eval/evaluation.py

@@ -1,104 +0,0 @@
-from pathlib import Path
-
-import yaml
-from pydantic import BaseModel, Field, model_validator
-
-from . import custom_evaluation
-from .answer_correctness import AnswerCorrectnessConfig
-from .llm_factory import Config as LLMConfig, create_llm, create_embedder
-from .steps.evaluation import evaluate_steps
-
-
-class Config(BaseModel):
-    llm: LLMConfig | None = None
-    custom_evaluations: list[custom_evaluation.Config] | None \
-        = Field(default=None, min_length=1)
-    answer_correctness: AnswerCorrectnessConfig | None = None
-
-    @model_validator(mode="after")
-    def validate_config(self) -> "Config":
-        if self.custom_evaluations and not self.llm:
-            msg = "llm config is required if custom_evaluations are provided"
-            raise ValueError(msg)
-        return self
-
-    @classmethod
-    def parse(cls, config_file_path: str | Path | None) -> "Config":
-        if config_file_path:
-            with open(config_file_path, encoding="utf-8") as f:
-                config_dict = yaml.safe_load(f)
-            return cls(**config_dict)
-        return cls()
-
-
-async def run_evaluation(
-    qa_dataset: list[dict],
-    responses_dict: dict,
-    config_file_path: str | Path | None = None,
-) -> list[dict]:
-    # Output metrics are not nested, for simpler aggregation
-    evaluation_results = []
-    config = Config.parse(config_file_path)
-    ragas_llm = create_llm(config)
-    ragas_embedder = create_embedder(config)
-    custom_evaluators = custom_evaluation.create_evaluators(config)
-    for template in qa_dataset:
-        template_id = template["template_id"]
-        for question in template["questions"]:
-            actual_result = responses_dict[question["id"]]
-            eval_result = {
-                "template_id": template_id,
-                "question_id": actual_result["question_id"],
-                "question_text": question["question_text"]
-            }
-            if "reference_answer" in question:
-                eval_result["reference_answer"] = question["reference_answer"]
-            if "reference_steps" in question:
-                eval_result["reference_steps"] = question["reference_steps"]
-            if "error" in actual_result:
-                eval_result.update({
-                    "status": "error",
-                    "error": actual_result["error"],
-                })
-            else:
-                eval_result["status"] = "success"
-
-            if "actual_answer" in actual_result:
-                eval_result["actual_answer"] = actual_result["actual_answer"]
-                if ragas_llm:
-                    from graphrag_eval.answer_relevance import Evaluator
-                    relevance_evaluator = Evaluator(ragas_llm, ragas_embedder)
-                    eval_result.update(
-                        await relevance_evaluator.get_relevance_dict(
-                            question["question_text"],
-                            actual_result["actual_answer"],
-                        )
-                    )
-                if "reference_answer" in question and ragas_llm:
-                    from graphrag_eval.answer_correctness import AnswerCorrectnessEvaluator
-                    answer_correctness_evaluator = AnswerCorrectnessEvaluator(
-                        llm=ragas_llm,
-                        config=config.answer_correctness,
-                    )
-                    eval_result.update(
-                        await answer_correctness_evaluator.get_correctness_dict(
-                            question,
-                            actual_result,
-                        )
-                    )
-            eval_result.update(
-                await evaluate_steps(
-                    question,
-                    actual_result,
-                    ragas_llm,
-                )
-            )
-            for custom_evaluator in custom_evaluators:
-                custom_metrics = await custom_evaluator.evaluate(question, actual_result)
-                eval_result.update(**custom_metrics)
-            for key in "input_tokens", "output_tokens", "total_tokens", "elapsed_sec":
-                if key in actual_result:
-                    eval_result[key] = actual_result[key]
-
-            evaluation_results.append(eval_result)
-    return evaluation_results