graphrag-eval 6.3.0__tar.gz → 6.5.0__tar.gz

{graphrag_eval-6.3.0 → graphrag_eval-6.5.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.3
 Name: graphrag-eval
-Version: 6.3.0
+Version: 6.5.0
 Summary: For assessing question answering systems' final answers and intermediate steps, against a given set of questions, reference answers and steps.
 License: Apache-2.0
 Author: Philip Ganchev
@@ -10,7 +10,7 @@ Classifier: License :: OSI Approved :: Apache Software License
 Classifier: Programming Language :: Python :: 3
 Classifier: Programming Language :: Python :: 3.12
 Provides-Extra: llm
-Requires-Dist: pydantic (==2.12.5)
+Requires-Dist: pydantic (==2.14.1)
 Requires-Dist: python-dateutil (==2.9.0.post0)
 Requires-Dist: ragas (==0.4.3) ; extra == "llm"
 Project-URL: Repository, https://github.com/Ontotext-AD/graphrag-eval

{graphrag_eval-6.3.0 → graphrag_eval-6.5.0}/graphrag_eval/aggregation.py

@@ -1,13 +1,13 @@
 import json
-import yaml
 from collections import defaultdict
 from collections.abc import Sequence
 from pathlib import Path
 from statistics import mean, median
 from typing import Any, Collection, Iterable
 
-from . import evaluation
+import yaml
 
+from . import evaluation
 
 METRICS = [
     "answer_recall",
@@ -155,7 +155,7 @@ def compute_micro_stats(
 ) -> dict:
     if custom_metrics is None:
         custom_metrics = []
-    
+
     values = number_of_samples_per_template_by_status.values()
     micro_summary = defaultdict(dict, {
         "number_of_error_samples": sum(v["error"] for v in values),
@@ -197,7 +197,7 @@ def compute_macro_stats(
 ) -> dict:
     if custom_metrics is None:
         custom_metrics = []
-    
+
     macro_summary = defaultdict(dict)
     for metric in METRICS + custom_metrics:
         means = [

graphrag_eval-6.5.0/graphrag_eval/answer_correctness.py

@@ -0,0 +1,172 @@
+from __future__ import annotations
+
+from pathlib import Path
+from typing import Any, Self, TYPE_CHECKING
+
+from pydantic import BaseModel, ConfigDict, Field, model_validator
+
+from graphrag_eval.util import compute_f1
+from .evaluator import Evaluator
+
+if TYPE_CHECKING:
+    from ragas.llms.base import InstructorBaseRagasLLM
+
+
+def load_default_prompt() -> str:
+    with open(
+        Path(__file__).parent / "prompts" / "template.md",
+        encoding="utf-8"
+    ) as f:
+        return f.read()
+
+
+class AnswerCorrectnessConfig(BaseModel):
+    enabled: bool = Field(default=True)
+    prompt: str = Field(default_factory=load_default_prompt)
+
+
+class AnswerCorrectnessResult(BaseModel):
+    model_config = ConfigDict(extra="forbid")
+
+    reference_claims_count: int = Field(ge=1)
+    actual_claims_count: int = Field(ge=1)
+    matching_claims_count: int = Field(ge=0)
+    reason: str = Field(min_length=1)
+
+    @model_validator(mode="after")
+    def validate_claim_counts(self) -> "AnswerCorrectnessResult":
+        if (
+            self.matching_claims_count > self.reference_claims_count or
+            self.matching_claims_count > self.actual_claims_count
+        ):
+            raise ValueError(
+                "matching_claims_count must be less than or equal to both "
+                "reference_claims_count and actual_claims_count"
+            )
+        return self
+
+
+class InvalidPromptException(Exception):
+    def __init__(
+        self,
+        message="The prompt template is invalid and cannot be "
+                "formatted."
+    ):
+        self.message = message
+        super().__init__(self.message)
+
+
+class AnswerCorrectnessEvaluator:
+    def __init__(
+        self,
+        ragas_llm: InstructorBaseRagasLLM,
+        config: AnswerCorrectnessConfig | None = None,
+    ):
+        self.config = config or AnswerCorrectnessConfig()
+        self.__validate_prompt_template(self.config.prompt)
+        self.prompt_template = self.config.prompt
+        self.ragas_llm = ragas_llm
+
+    @classmethod
+    def from_config(
+        cls,
+        ragas_llm: InstructorBaseRagasLLM | None,
+        config: AnswerCorrectnessConfig | None
+    ) -> Self | None:
+        if ragas_llm is None:
+            return None
+        if config is None or not config.enabled:
+            return None
+        return cls(ragas_llm=ragas_llm, config=config)
+
+    @staticmethod
+    def __validate_prompt_template(prompt_template: str):
+        try:
+            prompt_template.format(
+                question="Q?",
+                reference_answer="R",
+                actual_answer="A",
+            )
+        except Exception as exc:
+            raise InvalidPromptException(
+                "Invalid prompt template. Must only contain placeholders: "
+                "{question}, {reference_answer}, and {actual_answer}. "
+                f"Original error: {exc}"
+            ) from exc
+
+    async def _agenerate(self, prompt: str) -> AnswerCorrectnessResult:
+        """Wrapper method for easier testing"""
+        response = await self.ragas_llm.agenerate(prompt, AnswerCorrectnessResult)
+        return AnswerCorrectnessResult.model_validate(response)
+
+    async def evaluate_answer(
+        self,
+        question: str,
+        reference_answer: str,
+        actual_answer: str
+    ) -> AnswerCorrectnessResult:
+        if any(
+            not s.strip() for s in [question, reference_answer, actual_answer]
+        ):
+            raise ValueError(
+                "The question of the reference or the actual answer is a blank "
+                "string!"
+            )
+        prompt = self.prompt_template.format(
+            question=question,
+            reference_answer=reference_answer,
+            actual_answer=actual_answer,
+        )
+        return await self._agenerate(prompt)
+
+    async def evaluate(
+        self,
+        reference: dict[str, Any],
+        actual: dict[str, Any]
+    ) -> dict[str, Any]:
+        if "actual_answer" not in actual or "reference_answer" not in reference:
+            return {}
+        result = {}
+        try:
+            correctness = await self.evaluate_answer(
+                reference["question_text"],
+                reference["reference_answer"],
+                actual["actual_answer"],
+            )
+            result.update({
+                "answer_reference_claims_count": correctness.reference_claims_count,
+                "answer_actual_claims_count": correctness.actual_claims_count,
+                "answer_matching_claims_count": correctness.matching_claims_count,
+                "answer_correctness_reason": correctness.reason,
+            })
+            recall, precision, f1 = self.compute_recall_precision_f1(
+                correctness.reference_claims_count,
+                correctness.actual_claims_count,
+                correctness.matching_claims_count,
+            )
+            if recall is not None:
+                result["answer_recall"] = recall
+            if precision is not None:
+                result["answer_precision"] = precision
+            if f1 is not None:
+                result["answer_f1"] = f1
+        except Exception as exc:
+            result["answer_correctness_error"] = str(exc)
+        return result
+
+    @staticmethod
+    def compute_recall_precision_f1(
+        n_pos: int,
+        n_pred_pos: int,
+        n_true_pos: int,
+    ) -> tuple[float | None, float | None, float | None]:
+        recall = None
+        precision = None
+        if n_pos:
+            recall = n_true_pos / n_pos
+        if n_pred_pos:
+            precision = n_true_pos / n_pred_pos
+        return recall, precision, compute_f1(recall, precision)
+
+
+_: Evaluator = AnswerCorrectnessEvaluator

graphrag_eval-6.5.0/graphrag_eval/answer_relevance.py

@@ -0,0 +1,61 @@
+from __future__ import annotations
+
+from typing import Any, Self, TYPE_CHECKING
+
+from pydantic import BaseModel, Field
+
+from .evaluator import Evaluator
+
+if TYPE_CHECKING:
+    from ragas.llms.base import InstructorBaseRagasLLM
+    from ragas.embeddings.base import BaseRagasEmbeddings, BaseRagasEmbedding
+
+
+class AnswerRelevanceConfig(BaseModel):
+    enabled: bool = Field(default=True)
+
+
+class AnswerRelevanceEvaluator:
+    def __init__(
+        self,
+        ragas_llm: InstructorBaseRagasLLM,
+        ragas_embedder: BaseRagasEmbeddings | BaseRagasEmbedding
+    ):
+        from ragas.metrics.collections import AnswerRelevancy
+        self.scorer = AnswerRelevancy(llm=ragas_llm, embeddings=ragas_embedder)
+
+    @classmethod
+    def from_config(
+        cls,
+        ragas_llm: InstructorBaseRagasLLM | None,
+        ragas_embedder: BaseRagasEmbeddings | BaseRagasEmbedding | None,
+        config: AnswerRelevanceConfig | None
+    ) -> Self | None:
+        if ragas_llm is None or ragas_embedder is None:
+            return None
+        if config is None or not config.enabled:
+            return None
+        return cls(ragas_llm=ragas_llm, ragas_embedder=ragas_embedder)
+
+    async def evaluate(
+        self,
+        reference: dict[str, Any],
+        actual: dict[str, Any]
+    ) -> dict[str, Any]:
+        if "actual_answer" not in actual:
+            return {}
+        try:
+            result = await self.scorer.ascore(
+                user_input=reference["question_text"],
+                response=actual["actual_answer"]
+            )
+            return {
+                "answer_relevance": result.value
+            }
+        except Exception as e:
+            return {
+                "answer_relevance_error": str(e)
+            }
+
+
+_: Evaluator = AnswerRelevanceEvaluator

{graphrag_eval-6.3.0 → graphrag_eval-6.5.0}/graphrag_eval/cli/answer_correctness.py

@@ -1,8 +1,12 @@
+from __future__ import annotations
+
 import argparse
 import asyncio
 import csv
+import sys
 from argparse import ArgumentParser
 from pathlib import Path
+from typing import TYPE_CHECKING
 
 from tqdm import tqdm
 
@@ -10,34 +14,39 @@ from graphrag_eval import llm_factory
 from graphrag_eval.answer_correctness import AnswerCorrectnessEvaluator
 from graphrag_eval.evaluation import Config
 
+if TYPE_CHECKING:
+    from ragas.llms.base import InstructorBaseRagasLLM
+
 
 def parse_args() -> argparse.Namespace:
     parser = ArgumentParser(
-        description="Calculates answer correctness over the entries from the input tsv file and "
-                    "stores the output in the output tsv file.",
+        description="Calculates answer correctness over the entries from the "
+                    "input tsv file and stores the output in the output tsv "
+                    "file.",
     )
     parser.add_argument(
         "-i",
         "--input-tsv-file-path",
         type=Path,
         required=True,
-        help="Input tsv file path with columns `Question`, `Reference answer` and `Actual answer`",
+        help="Input tsv file path with columns `Question`, `Reference answer` "
+             "and `Actual answer`",
     )
     parser.add_argument(
         "-o",
         "--output-tsv-file-path",
         type=Path,
         required=True,
-        help="Output tsv file path with columns `#Reference`, `#PTarget`, `#Matching`, "
-             "`Reasoning`, `Error`",
+        help="Output tsv file path with columns `#Reference`, `#PTarget`, "
+             "`#Matching`, `Reasoning`, `Error`",
     )
     parser.add_argument(
         "-c",
         "--config-yaml-file-path",
         type=Path,
         required=True,
-        help="Config yaml file path with definition of the LLM to use and optionally a custom "
-             "prompt.",
+        help="Config yaml file path with definition of the LLM to use and "
+             "optionally a custom prompt.",
     )
     return parser.parse_args()
 
@@ -47,6 +56,8 @@ async def evaluate_and_write(
     output_tsv_file_path: Path,
     evaluator: AnswerCorrectnessEvaluator,
 ) -> None:
+    csv.field_size_limit(sys.maxsize)
+
     with open(input_tsv_file_path, encoding="utf-8") as f:
         reader = csv.DictReader(f, delimiter="\t")
         rows = [row for row in reader]
@@ -54,7 +65,9 @@ async def evaluate_and_write(
     output_tsv_file_path.parent.mkdir(parents=True, exist_ok=True)
     with open(output_tsv_file_path, "w", encoding="utf-8") as f:
         writer = csv.writer(f, delimiter="\t")
-        writer.writerow(["#Reference", "#PTarget", "#Matching", "Reasoning", "Error"])
+        writer.writerow(
+            ["#Reference", "#PTarget", "#Matching", "Reasoning", "Error"]
+        )
 
         for row in tqdm(rows):
             if "Question" not in row or \
@@ -63,13 +76,18 @@ async def evaluate_and_write(
                 raise ValueError("Unexpected input format!")
 
             try:
-                vals = await evaluator.evaluate_answer(
+                correctness = await evaluator.evaluate_answer(
                     row["Question"],
                     row["Reference answer"],
                     row["Actual answer"]
                 )
-                vals = vals + ("",)
-                writer.writerow(vals)
+                writer.writerow([
+                    correctness.reference_claims_count,
+                    correctness.actual_claims_count,
+                    correctness.matching_claims_count,
+                    correctness.reason,
+                    "",
+                ])
             except Exception as exc:
                 writer.writerow(["", "", "", "", str(exc)])
             f.flush()
@@ -81,19 +99,26 @@ def run(
     output_tsv_file_path: Path,
 ):
     config = Config.parse(config_yaml_file_path)
-    ragas_llm = llm_factory.create_llm(config)
+    ragas_llm: InstructorBaseRagasLLM | None = llm_factory.create_llm(
+        config.llm
+    )
     if ragas_llm is None:
-        raise ValueError("LLM must be configured to calculate the answer correctness!")
-    else:
-        evaluator = AnswerCorrectnessEvaluator(
-            llm=ragas_llm,
-            config=config.answer_correctness,
+        raise ValueError(
+            "LLM must be configured to calculate the answer correctness!"
+        )
+    if config.answer_correctness and not config.answer_correctness.enabled:
+        raise ValueError(
+            "Can't disable answer correctness, when running this script!"
         )
-        asyncio.run(evaluate_and_write(
-            input_tsv_file_path,
-            output_tsv_file_path,
-            evaluator,
-        ))
+    evaluator = AnswerCorrectnessEvaluator(
+        ragas_llm=ragas_llm,
+        config=config.answer_correctness,
+    )
+    asyncio.run(evaluate_and_write(
+        input_tsv_file_path,
+        output_tsv_file_path,
+        evaluator,
+    ))
 
 
 def main():

{graphrag_eval-6.3.0 → graphrag_eval-6.5.0}/graphrag_eval/custom_evaluation.py

@@ -1,9 +1,14 @@
+from __future__ import annotations
+
 import json
-from typing import Literal
+from typing import Literal, Self, TYPE_CHECKING, Any
 
 from pydantic import BaseModel, ConfigDict, Field, model_validator
 
-from graphrag_eval.llm_factory import create_llm
+from .evaluator import Evaluator
+
+if TYPE_CHECKING:
+    from ragas.llms.base import InstructorBaseRagasLLM
 
 RESERVED_KEYS = {
     "template_id",
@@ -43,7 +48,7 @@ Inputs = Literal[
 StepsKey = Literal["args", "output"]
 
 
-class Config(BaseModel):
+class EvaluatorConfig(BaseModel):
     model_config = ConfigDict(extra='forbid')
     name: str
     inputs: list[Inputs] = Field(..., min_length=1)
@@ -53,7 +58,7 @@ class Config(BaseModel):
     steps_keys: set[StepsKey] | None = Field(default=None, min_length=1)
 
     @model_validator(mode='after')
-    def validate_step_dependencies(self) -> 'Config':
+    def validate_step_dependencies(self) -> Self:
         if set(self.inputs) & {"reference_steps", "actual_steps"}:
             suffix = "is required when steps are in inputs"
             for var_name in ["steps_name", "steps_keys"]:
@@ -62,7 +67,7 @@ class Config(BaseModel):
         return self
 
     @model_validator(mode='after')
-    def validate_name_and_outputs(self) -> 'Config':
+    def validate_name_and_outputs(self) -> Self:
         if self.name + "_error" in RESERVED_KEYS:
             raise ValueError(f"Name {self.name} is reserved")
         conflicting_keys = set(self.outputs.keys()) & RESERVED_KEYS
@@ -76,7 +81,7 @@ def create_input_template(input_key: str) -> str:
     return f"# {header}\n{{{input_key}}}"
 
 
-def create_prompt_template(config: Config, output_variables: list[str]) -> str:
+def create_prompt_template(config: EvaluatorConfig, output_variables: list[str]) -> str:
     """
     Return a template for the LLM prompt, with placeholders for the inputs, 
     instructions, outputs etc. We use this template at evaluation time to
@@ -99,8 +104,8 @@ def create_prompt_template(config: Config, output_variables: list[str]) -> str:
 class CustomEvaluator:
     def __init__(
         self,
-        config: Config,
-        eval_config: "evaluation.Config",
+        ragas_llm: InstructorBaseRagasLLM,
+        config: EvaluatorConfig,
     ):
         self.name = config.name
         self.input_variables = config.inputs
@@ -111,11 +116,24 @@ class CustomEvaluator:
             config,
             self.output_variables
         )
-        self.llm = create_llm(eval_config)
+        self.ragas_llm = ragas_llm
+
+    @classmethod
+    def from_config(
+        cls,
+        ragas_llm: InstructorBaseRagasLLM | None,
+        evaluation_configs: list[EvaluatorConfig] | None
+    ) -> list[Self]:
+        if ragas_llm and evaluation_configs:
+            return [
+                cls(ragas_llm, evaluation_config)
+                for evaluation_config in evaluation_configs
+            ]
+        return []
 
     async def _agenerate(self, prompt: str) -> str:
         """Wrapper method for easier testing"""
-        return (await self.llm.agenerate(prompt, None)).choices[0].message.content
+        return (await self.ragas_llm.agenerate(prompt, None)).choices[0].message.content
 
     def format_steps(self, steps: list) -> str:
         steps_formatted = []
@@ -157,7 +175,11 @@ class CustomEvaluator:
             return result
         return self.error(f"Expected {n_exp} tab-separated values, got: {response}")
 
-    async def evaluate(self, reference: dict, actual: dict) -> dict[str, str | None]:
+    async def evaluate(
+        self,
+        reference: dict[str, Any],
+        actual: dict[str, Any]
+    ) -> dict[str, Any]:
         inputs = {}
         if "question" in self.input_variables:
             if "question_text" not in reference:
@@ -195,10 +217,4 @@ class CustomEvaluator:
         return self.parse_outputs(response)
 
 
-def create_evaluators(config: "evaluation.Config") -> list[CustomEvaluator]:
-    if config.custom_evaluations and config.llm:
-        return [
-            CustomEvaluator(custom_evaluation_config, config)
-            for custom_evaluation_config in config.custom_evaluations
-        ]
-    return []
+_: Evaluator = CustomEvaluator

graphrag_eval-6.5.0/graphrag_eval/evaluation.py

@@ -0,0 +1,154 @@
+from __future__ import annotations
+
+from pathlib import Path
+from typing import Self, TYPE_CHECKING
+
+import yaml
+from pydantic import BaseModel, Field, model_validator
+
+from .answer_correctness import (
+    AnswerCorrectnessConfig,
+    AnswerCorrectnessEvaluator,
+)
+from .answer_relevance import AnswerRelevanceConfig, AnswerRelevanceEvaluator
+from .custom_evaluation import EvaluatorConfig, CustomEvaluator
+from .evaluator import Evaluator
+from .llm_factory import LLMConfig, create_llm, create_embedder
+from .steps.evaluation import evaluate_steps
+
+if TYPE_CHECKING:
+    from ragas.llms.base import InstructorBaseRagasLLM
+    from ragas.embeddings.base import BaseRagasEmbeddings, BaseRagasEmbedding
+
+
+class Config(BaseModel):
+    llm: LLMConfig | None = None
+    custom_evaluations: list[EvaluatorConfig] | None = Field(
+        default=None,
+        min_length=1
+    )
+    answer_correctness: AnswerCorrectnessConfig | None = None
+    answer_relevance: AnswerRelevanceConfig | None = None
+
+    @model_validator(mode="after")
+    def validate_config_and_set_defaults(self) -> Self:
+        has_llm = self.llm is not None
+        has_embedding = has_llm and self.llm.embedding is not None
+
+        if self.answer_correctness is None and has_llm:
+            self.answer_correctness = AnswerCorrectnessConfig()
+
+        if self.answer_relevance is None and has_embedding:
+            self.answer_relevance = AnswerRelevanceConfig()
+
+        if self.custom_evaluations and not has_llm:
+            raise ValueError(
+                "llm config is required if custom_evaluations are provided"
+            )
+        if (
+            self.answer_correctness
+            and self.answer_correctness.enabled
+            and not has_llm
+        ):
+            raise ValueError(
+                "llm config is required if answer correctness is enabled"
+            )
+        if (
+            self.answer_relevance
+            and self.answer_relevance.enabled
+            and not has_embedding
+        ):
+            raise ValueError(
+                "llm config including embedding is required if answer "
+                "relevance is enabled"
+            )
+        return self
+
+    @classmethod
+    def parse(cls, config_file_path: str | Path | None) -> Self:
+        if config_file_path:
+            with open(config_file_path, encoding="utf-8") as f:
+                config_dict = yaml.safe_load(f)
+            return cls(**config_dict)
+        return cls()
+
+
+async def run_evaluation(
+    qa_dataset: list[dict],
+    responses_dict: dict,
+    config_file_path: str | Path | None = None,
+) -> list[dict]:
+    evaluators, ragas_llm = parse_config_and_init_evaluators(config_file_path)
+
+    # Output metrics are not nested, for simpler aggregation
+    evaluation_results = []
+    for template in qa_dataset:
+        template_id = template["template_id"]
+        for question in template["questions"]:
+            actual_result = responses_dict[question["id"]]
+            eval_result = {
+                "template_id": template_id,
+                "question_id": actual_result["question_id"],
+                "question_text": question["question_text"]
+            }
+            for key in ("input_tokens", "output_tokens", "total_tokens",
+                        "elapsed_sec"):
+                if key in actual_result:
+                    eval_result[key] = actual_result[key]
+            if "actual_answer" in actual_result:
+                eval_result["actual_answer"] = actual_result["actual_answer"]
+            if "reference_answer" in question:
+                eval_result["reference_answer"] = question["reference_answer"]
+            if "reference_steps" in question:
+                eval_result["reference_steps"] = question["reference_steps"]
+            if "error" in actual_result:
+                eval_result.update({
+                    "status": "error",
+                    "error": actual_result["error"],
+                })
+            else:
+                eval_result["status"] = "success"
+
+            eval_result.update(
+                await evaluate_steps(question, actual_result, ragas_llm)
+            )
+            for evaluator in evaluators:
+                eval_result.update(
+                    await evaluator.evaluate(question, actual_result)
+                )
+
+            evaluation_results.append(eval_result)
+    return evaluation_results
+
+
+def parse_config_and_init_evaluators(
+    config_file_path: str | Path | None
+) -> tuple[
+    list[Evaluator],
+    InstructorBaseRagasLLM | None,
+]:
+    config = Config.parse(config_file_path)
+    ragas_llm: InstructorBaseRagasLLM | None = create_llm(config.llm)
+    ragas_embedder: BaseRagasEmbeddings | BaseRagasEmbedding | None = (
+        create_embedder(config.llm)
+    )
+
+    evaluators: list[Evaluator] = []
+
+    answer_relevance_evaluator = AnswerRelevanceEvaluator.from_config(
+        ragas_llm, ragas_embedder, config.answer_relevance
+    )
+    if answer_relevance_evaluator:
+        evaluators.append(answer_relevance_evaluator)
+
+    answer_correctness_evaluator = AnswerCorrectnessEvaluator.from_config(
+        ragas_llm, config.answer_correctness
+    )
+    if answer_correctness_evaluator:
+        evaluators.append(answer_correctness_evaluator)
+
+    evaluators.extend(
+        CustomEvaluator.from_config(ragas_llm, config.custom_evaluations)
+    )
+
+    return evaluators, ragas_llm

graphrag_eval-6.5.0/graphrag_eval/evaluator.py

@@ -0,0 +1,14 @@
+from typing import Protocol, Any
+
+
+class Evaluator(Protocol):
+    async def evaluate(
+        self,
+        reference: dict[str, Any],
+        actual: dict[str, Any]
+    ) -> dict[str, Any]:
+        """
+        Evaluate the actual output against the reference.
+        Returns a flat dictionary containing scores or error tracking logs.
+        """
+        ...

{graphrag_eval-6.3.0 → graphrag_eval-6.5.0}/graphrag_eval/llm_factory.py

@@ -1,7 +1,13 @@
-from typing import Optional
+from __future__ import annotations
+
+from typing import TYPE_CHECKING
 
 from pydantic import BaseModel, ConfigDict, Field
 
+if TYPE_CHECKING:
+    from ragas.llms.base import InstructorBaseRagasLLM
+    from ragas.embeddings.base import BaseRagasEmbeddings, BaseRagasEmbedding
+
 
 class GenerationConfig(BaseModel):
     provider: str
@@ -17,18 +23,20 @@ class EmbeddingConfig(BaseModel):
     model_config = ConfigDict(extra='allow')
 
 
-class Config(BaseModel):
+class LLMConfig(BaseModel):
     generation: GenerationConfig
     embedding: EmbeddingConfig | None = None
 
 
-def create_llm(config: "evaluation.Config") -> Optional["InstructorBaseRagasLLM"]:
-    if config.llm:
+def create_llm(
+    config: LLMConfig | None
+) -> InstructorBaseRagasLLM | None:
+    if config:
         import litellm
         from ragas.llms import llm_factory
 
         litellm.drop_params = True  # Remove unsupported params from requests
-        params = config.llm.generation.model_dump()
+        params = config.generation.model_dump()
         ragas_llm = llm_factory(
             provider="litellm",
             model=f"{params.pop('provider')}/{params.pop('model')}",
@@ -40,13 +48,15 @@ def create_llm(config: "evaluation.Config") -> Optional["InstructorBaseRagasLLM"
     return None
 
 
-def create_embedder(config: "evaluation.Config") -> Optional["BaseRagasEmbedding"]:
-    if config.llm and config.llm.embedding:
+def create_embedder(
+    config: LLMConfig | None
+) -> BaseRagasEmbeddings | BaseRagasEmbedding | None:
+    if config and config.embedding:
         import litellm
         from ragas.embeddings.base import embedding_factory
 
         litellm.drop_params = True  # Remove unsupported params from requests
-        params = config.llm.embedding.model_dump()
+        params = config.embedding.model_dump()
         ragas_embedder = embedding_factory(
             provider="litellm",
             model=f"{params.pop('provider')}/{params.pop('model')}",

graphrag_eval-6.5.0/graphrag_eval/prompts/template.md

@@ -0,0 +1,31 @@
+Below are a query, a reference response and a candidate response to it.
+1. Extract all claims from each response.
+2. Find matching claims between responses. Matching claims have the same meaning and details such as subjects, names, locations, amounts, IDs, commands and paths.
+3. Output one JSON object only. Do not include Markdown, code fences, or any text outside the JSON object.
+
+# Query
+{question}
+
+# Reference response
+{reference_answer}
+
+# Candidate response
+{actual_answer}
+
+# JSON fields
+* reference_claims_count: Count of reference response claims
+* actual_claims_count: Count of candidate response claims
+* matching_claims_count: Count of matching claims
+* reason: Explanation of the counts in English
+
+# Value checks
+* 1 <= reference_claims_count, actual_claims_count
+* 0 <= matching_claims_count <= reference_claims_count, actual_claims_count
+
+# Output format
+{{
+  "reference_claims_count": 1,
+  "actual_claims_count": 1,
+  "matching_claims_count": 1,
+  "reason": "The reference and candidate responses contain the same single claim."
+}}

{graphrag_eval-6.3.0 → graphrag_eval-6.5.0}/graphrag_eval/steps/evaluation.py

@@ -1,13 +1,21 @@
+from __future__ import annotations
+
 import json
 import logging
 from collections import defaultdict
 from collections.abc import Sequence
-from typing import Any
+from typing import Any, TYPE_CHECKING
 
 from .iri_discovery import do_iri_discovery_steps_equal
 from .retrieval_context_ids import recall_at_k
 from .sparql import compare_sparql_results
-from .timeseries import do_retrieve_time_series_steps_equal, do_retrieve_data_points_steps_equal
+from .timeseries import (
+    do_retrieve_time_series_steps_equal,
+    do_retrieve_data_points_steps_equal,
+)
+
+if TYPE_CHECKING:
+    from ragas.llms.base import InstructorBaseRagasLLM
 
 logger = logging.getLogger(__name__)
 
@@ -140,7 +148,7 @@ def calculate_steps_score(
 async def evaluate_steps(
     reference: dict,
     actual: dict,
-    ragas_llm: "InstructorBaseRagasLLM",
+    ragas_llm: InstructorBaseRagasLLM | None,
 ) -> dict:
     eval_result = {}
     actual_steps = actual.get("actual_steps", [])

{graphrag_eval-6.3.0 → graphrag_eval-6.5.0}/pyproject.toml

@@ -1,6 +1,6 @@
 [project]
 name = "graphrag-eval"
-version = "6.3.0"
+version = "6.5.0"
 description = "For assessing question answering systems' final answers and intermediate steps, against a given set of questions, reference answers and steps."
 authors = [
     { name = "Philip Ganchev", email = "philip.ganchev@graphwise.ai" },
@@ -17,23 +17,23 @@ repository = "https://github.com/Ontotext-AD/graphrag-eval"
 [tool.poetry.dependencies]
 python-dateutil = "2.9.0.post0"
 ragas = { version = "0.4.3", optional = true }
-pydantic = "2.12.5"
+pydantic = "2.14.1"
 
 [tool.poetry.extras]
 llm = ["ragas", "litellm", "pyyaml"]
 
 [tool.poetry.group.llm.dependencies]
 ragas = "0.4.3"
-litellm = "1.85.1"
+litellm = "1.105.1"
 pyyaml = "6.0.3"
 
 [tool.poetry.group.llm]
 optional = true
 
 [tool.poetry.group.test.dependencies]
-pytest = "<10,>=9.0.3"
-pytest-asyncio = "1.3.0"
-pytest-cov = "<8,>=7"
+pytest = "<10,>=9.1.1"
+pytest-asyncio = "<2,>=1.4.0"
+pytest-cov = "<8,>=7.1.0"
 jsonlines = "4.0.0"
 pyyaml = "6.0.3"
 

graphrag_eval-6.3.0/graphrag_eval/answer_correctness.py

@@ -1,139 +0,0 @@
-from pathlib import Path
-
-from pydantic import BaseModel, Field
-
-from graphrag_eval.util import compute_f1
-
-
-def load_default_prompt() -> str:
-    with open(Path(__file__).parent / "prompts" / "template.md", "r", encoding="utf-8") as f:
-        return f.read()
-
-
-class AnswerCorrectnessConfig(BaseModel):
-    prompt: str = Field(default_factory=load_default_prompt)
-
-
-class InvalidPromptException(Exception):
-    def __init__(self, message="The prompt template is invalid and cannot be formatted."):
-        self.message = message
-        super().__init__(self.message)
-
-
-class AnswerCorrectnessEvaluator:
-    def __init__(
-        self,
-        llm: "InstructorBaseRagasLLM",
-        config: AnswerCorrectnessConfig | None = None,
-    ):
-        self.config = config or AnswerCorrectnessConfig()
-        self.__validate_prompt_template(self.config.prompt)
-        self.prompt_template = self.config.prompt
-        self.llm = llm
-
-    @staticmethod
-    def __validate_prompt_template(prompt_template: str):
-        try:
-            prompt_template.format(
-                question="Q?",
-                reference_answer="R",
-                actual_answer="A",
-            )
-        except Exception as exc:
-            raise InvalidPromptException(
-                "Invalid prompt template. Must only contain placeholders: "
-                "{question}, {reference_answer}, and {actual_answer}. "
-                f"Original error: {exc}"
-            ) from exc
-
-    async def _agenerate(self, prompt):
-        """Wrapper method for easier testing"""
-        return (await self.llm.agenerate(prompt, None)).choices[0].message.content
-
-    async def evaluate_answer(
-        self,
-        question: str,
-        reference_answer: str,
-        actual_answer: str
-    ) -> tuple[int, int, int, str]:
-        if any(not s.strip() for s in [question, reference_answer, actual_answer]):
-            raise ValueError("The question of the reference or the actual answer is a blank "
-                             "string!")
-        prompt = self.prompt_template.format(
-            question=question,
-            reference_answer=reference_answer,
-            actual_answer=actual_answer,
-        )
-        response_str = await self._agenerate(prompt)
-        return self.extract_response_values(response_str)
-
-    async def get_correctness_dict(
-        self,
-        reference: dict,
-        actual: dict,
-    ):
-        result = {"reference_answer": reference["reference_answer"]}
-        try:
-            num_ref_claims, num_actual_claims, num_matching_claims, reason = \
-                await self.evaluate_answer(
-                    reference["question_text"],
-                    reference["reference_answer"],
-                    actual["actual_answer"],
-                )
-            result.update({
-                "answer_reference_claims_count": num_ref_claims,
-                "answer_actual_claims_count": num_actual_claims,
-                "answer_matching_claims_count": num_matching_claims,
-                "answer_correctness_reason": reason,
-            })
-            recall, precision, f1 = self.compute_recall_precision_f1(
-                num_ref_claims, num_actual_claims, num_matching_claims
-            )
-            if recall is not None:
-                result["answer_recall"] = recall
-            if precision is not None:
-                result["answer_precision"] = precision
-            if f1 is not None:
-                result["answer_f1"] = f1
-        except Exception as exc:
-            result["answer_eval_error"] = str(exc)
-        return result
-
-    @staticmethod
-    def compute_recall_precision_f1(
-        n_pos: int,
-        n_pred_pos: int,
-        n_true_pos: int,
-    ) -> tuple[float | None, float | None, float | None]:
-        recall = None
-        precision = None
-        if n_pos:
-            recall = n_true_pos / n_pos
-        if n_pred_pos:
-            precision = n_true_pos / n_pred_pos
-        return recall, precision, compute_f1(recall, precision)
-
-    @staticmethod
-    def extract_response_values(
-        response: str
-    ) -> tuple[int, int, int, str]:
-        vals = response.split("\t")
-        n = len(vals)
-        if n < 4:
-            raise ValueError(f"Expected 4 tab-separated values: {response}")
-        vals = vals[:4]
-        try:
-            n_ref, n_actual, n_matching = map(int, vals[:3])
-        except ValueError:
-            raise ValueError(f"Claims counts should be ints: {vals}")
-        if any([
-            n_ref < 1,
-            n_actual < 1,
-            n_matching < 0,
-            n_matching > n_ref,
-            n_matching > n_actual
-        ]):
-            raise ValueError(
-                f"Invalid claims counts combination: {n_ref}\t{n_actual}\t{n_matching}"
-            )
-        return n_ref, n_actual, n_matching, vals[3]

graphrag_eval-6.3.0/graphrag_eval/answer_relevance.py

@@ -1,29 +0,0 @@
-from ragas.embeddings.base import BaseRagasEmbedding
-from ragas.llms.base import InstructorBaseRagasLLM
-from ragas.metrics.collections import AnswerRelevancy
-
-from graphrag_eval.util import singleton
-
-
-@singleton
-class Evaluator:
-    def __init__(self, ragas_llm: InstructorBaseRagasLLM, ragas_embedder: BaseRagasEmbedding):
-        self.scorer = AnswerRelevancy(llm=ragas_llm, embeddings=ragas_embedder)
-
-    async def get_relevance_dict(
-        self,
-        question_text: str,
-        actual_answer: str,
-    ) -> dict:
-        try:
-            result = await self.scorer.ascore(
-                user_input=question_text,
-                response=actual_answer
-            )
-            return {
-                "answer_relevance": result.value
-            }
-        except Exception as e:
-            return {
-                "answer_relevance_error": str(e)
-            }

graphrag_eval-6.3.0/graphrag_eval/evaluation.py

@@ -1,104 +0,0 @@
-from pathlib import Path
-
-import yaml
-from pydantic import BaseModel, Field, model_validator
-
-from . import custom_evaluation
-from .answer_correctness import AnswerCorrectnessConfig
-from .llm_factory import Config as LLMConfig, create_llm, create_embedder
-from .steps.evaluation import evaluate_steps
-
-
-class Config(BaseModel):
-    llm: LLMConfig | None = None
-    custom_evaluations: list[custom_evaluation.Config] | None \
-        = Field(default=None, min_length=1)
-    answer_correctness: AnswerCorrectnessConfig | None = None
-
-    @model_validator(mode="after")
-    def validate_config(self) -> "Config":
-        if self.custom_evaluations and not self.llm:
-            msg = "llm config is required if custom_evaluations are provided"
-            raise ValueError(msg)
-        return self
-
-    @classmethod
-    def parse(cls, config_file_path: str | Path | None) -> "Config":
-        if config_file_path:
-            with open(config_file_path, encoding="utf-8") as f:
-                config_dict = yaml.safe_load(f)
-            return cls(**config_dict)
-        return cls()
-
-
-async def run_evaluation(
-    qa_dataset: list[dict],
-    responses_dict: dict,
-    config_file_path: str | Path | None = None,
-) -> list[dict]:
-    # Output metrics are not nested, for simpler aggregation
-    evaluation_results = []
-    config = Config.parse(config_file_path)
-    ragas_llm = create_llm(config)
-    ragas_embedder = create_embedder(config)
-    custom_evaluators = custom_evaluation.create_evaluators(config)
-    for template in qa_dataset:
-        template_id = template["template_id"]
-        for question in template["questions"]:
-            actual_result = responses_dict[question["id"]]
-            eval_result = {
-                "template_id": template_id,
-                "question_id": actual_result["question_id"],
-                "question_text": question["question_text"]
-            }
-            if "reference_answer" in question:
-                eval_result["reference_answer"] = question["reference_answer"]
-            if "reference_steps" in question:
-                eval_result["reference_steps"] = question["reference_steps"]
-            if "error" in actual_result:
-                eval_result.update({
-                    "status": "error",
-                    "error": actual_result["error"],
-                })
-            else:
-                eval_result["status"] = "success"
-
-            if "actual_answer" in actual_result:
-                eval_result["actual_answer"] = actual_result["actual_answer"]
-                if ragas_llm:
-                    from graphrag_eval.answer_relevance import Evaluator
-                    relevance_evaluator = Evaluator(ragas_llm, ragas_embedder)
-                    eval_result.update(
-                        await relevance_evaluator.get_relevance_dict(
-                            question["question_text"],
-                            actual_result["actual_answer"],
-                        )
-                    )
-                if "reference_answer" in question and ragas_llm:
-                    from graphrag_eval.answer_correctness import AnswerCorrectnessEvaluator
-                    answer_correctness_evaluator = AnswerCorrectnessEvaluator(
-                        llm=ragas_llm,
-                        config=config.answer_correctness,
-                    )
-                    eval_result.update(
-                        await answer_correctness_evaluator.get_correctness_dict(
-                            question,
-                            actual_result,
-                        )
-                    )
-            eval_result.update(
-                await evaluate_steps(
-                    question,
-                    actual_result,
-                    ragas_llm,
-                )
-            )
-            for custom_evaluator in custom_evaluators:
-                custom_metrics = await custom_evaluator.evaluate(question, actual_result)
-                eval_result.update(**custom_metrics)
-            for key in "input_tokens", "output_tokens", "total_tokens", "elapsed_sec":
-                if key in actual_result:
-                    eval_result[key] = actual_result[key]
-
-            evaluation_results.append(eval_result)
-    return evaluation_results

graphrag_eval-6.3.0/graphrag_eval/prompts/template.md

@@ -1,26 +0,0 @@
-Below are a query, a reference response and a candidate response to it.
-1. Extract all claims from each response
-2. Find matching claims between responses. Matching claims have the same meaning and details such as subjects, names, locations, amounts, IDs, commands and paths.
-3. Output the values listed below (all and only those).
-
-# Query
-{question}
-
-# Reference response
-{reference_answer}
-
-# Candidate response
-{actual_answer}
-
-# Output values
-* v1: Count of reference response claims
-* v2: Count of candidate response claims
-* v3: Count of matching claims
-* v4: Explanation of v1-v3 in English
-
-# Value checks
-* 1 <= v1, v2
-* 0 <= v3 <= v1, v2
-
-# Output format
-<v1><tab><v2><tab><v3><tab><v4>