graphrag-eval 6.1.0__tar.gz → 6.3.0__tar.gz

graphrag_eval-6.3.0/PKG-INFO

@@ -0,0 +1,46 @@
+Metadata-Version: 2.3
+Name: graphrag-eval
+Version: 6.3.0
+Summary: For assessing question answering systems' final answers and intermediate steps, against a given set of questions, reference answers and steps.
+License: Apache-2.0
+Author: Philip Ganchev
+Author-email: philip.ganchev@graphwise.ai
+Requires-Python: >=3.12,<3.13
+Classifier: License :: OSI Approved :: Apache Software License
+Classifier: Programming Language :: Python :: 3
+Classifier: Programming Language :: Python :: 3.12
+Provides-Extra: llm
+Requires-Dist: pydantic (==2.12.5)
+Requires-Dist: python-dateutil (==2.9.0.post0)
+Requires-Dist: ragas (==0.4.3) ; extra == "llm"
+Project-URL: Repository, https://github.com/Ontotext-AD/graphrag-eval
+Description-Content-Type: text/markdown
+
+<p align="center">
+  <img alt="Graphwise Logo" src="https://github.com/Ontotext-AD/graphrag-eval/blob/main/.github/Graphwise_Logo.jpg">
+</p>
+
+# QA Evaluation
+
+This is a Python library for assessing the quality of question-answering systems, such as systems built with LLM-based agents. It is agnostic to the agent implementation and the LLM it uses.
+
+The evaluation is based on a user-provided reference dataset containing queries, reference responses, and optional reference steps, such as expected tool uses. The evaluator compares these references with the agent's actual responses and executed steps. Reference steps can be grouped to allow some expected steps to occur in any order.
+
+The library provides built-in evaluation metrics and supports user-defined custom metrics ([§ Metrics](https://github.com/Ontotext-AD/graphrag-eval/blob/main/docs/metrics.md)).
+
+## Documentation
+
+- [Quickstart](https://github.com/Ontotext-AD/graphrag-eval/blob/main/docs/quickstart.md)
+- [Metrics](https://github.com/Ontotext-AD/graphrag-eval/blob/main/docs/metrics.md)
+- [Configuration](https://github.com/Ontotext-AD/graphrag-eval/blob/main/docs/config.md)
+- [Input](https://github.com/Ontotext-AD/graphrag-eval/blob/main/docs/input.md)
+- [Output](https://github.com/Ontotext-AD/graphrag-eval/blob/main/docs/output.md)
+
+## Maintainers
+
+Developed and maintained by [Graphwise](https://graphwise.ai/). For issues and feature requests, please open a [GitHub issue](https://github.com/Ontotext-AD/graphrag-eval/issues).
+
+## License
+
+Apache-2.0 License. See the [LICENSE](https://github.com/Ontotext-AD/graphrag-eval/blob/main/LICENSE) file for details.
+

graphrag_eval-6.3.0/README.md

@@ -0,0 +1,27 @@
+<p align="center">
+  <img alt="Graphwise Logo" src="https://github.com/Ontotext-AD/graphrag-eval/blob/main/.github/Graphwise_Logo.jpg">
+</p>
+
+# QA Evaluation
+
+This is a Python library for assessing the quality of question-answering systems, such as systems built with LLM-based agents. It is agnostic to the agent implementation and the LLM it uses.
+
+The evaluation is based on a user-provided reference dataset containing queries, reference responses, and optional reference steps, such as expected tool uses. The evaluator compares these references with the agent's actual responses and executed steps. Reference steps can be grouped to allow some expected steps to occur in any order.
+
+The library provides built-in evaluation metrics and supports user-defined custom metrics ([§ Metrics](https://github.com/Ontotext-AD/graphrag-eval/blob/main/docs/metrics.md)).
+
+## Documentation
+
+- [Quickstart](https://github.com/Ontotext-AD/graphrag-eval/blob/main/docs/quickstart.md)
+- [Metrics](https://github.com/Ontotext-AD/graphrag-eval/blob/main/docs/metrics.md)
+- [Configuration](https://github.com/Ontotext-AD/graphrag-eval/blob/main/docs/config.md)
+- [Input](https://github.com/Ontotext-AD/graphrag-eval/blob/main/docs/input.md)
+- [Output](https://github.com/Ontotext-AD/graphrag-eval/blob/main/docs/output.md)
+
+## Maintainers
+
+Developed and maintained by [Graphwise](https://graphwise.ai/). For issues and feature requests, please open a [GitHub issue](https://github.com/Ontotext-AD/graphrag-eval/issues).
+
+## License
+
+Apache-2.0 License. See the [LICENSE](https://github.com/Ontotext-AD/graphrag-eval/blob/main/LICENSE) file for details.

graphrag_eval-6.3.0/graphrag_eval/answer_correctness.py

@@ -0,0 +1,139 @@
+from pathlib import Path
+
+from pydantic import BaseModel, Field
+
+from graphrag_eval.util import compute_f1
+
+
+def load_default_prompt() -> str:
+    with open(Path(__file__).parent / "prompts" / "template.md", "r", encoding="utf-8") as f:
+        return f.read()
+
+
+class AnswerCorrectnessConfig(BaseModel):
+    prompt: str = Field(default_factory=load_default_prompt)
+
+
+class InvalidPromptException(Exception):
+    def __init__(self, message="The prompt template is invalid and cannot be formatted."):
+        self.message = message
+        super().__init__(self.message)
+
+
+class AnswerCorrectnessEvaluator:
+    def __init__(
+        self,
+        llm: "InstructorBaseRagasLLM",
+        config: AnswerCorrectnessConfig | None = None,
+    ):
+        self.config = config or AnswerCorrectnessConfig()
+        self.__validate_prompt_template(self.config.prompt)
+        self.prompt_template = self.config.prompt
+        self.llm = llm
+
+    @staticmethod
+    def __validate_prompt_template(prompt_template: str):
+        try:
+            prompt_template.format(
+                question="Q?",
+                reference_answer="R",
+                actual_answer="A",
+            )
+        except Exception as exc:
+            raise InvalidPromptException(
+                "Invalid prompt template. Must only contain placeholders: "
+                "{question}, {reference_answer}, and {actual_answer}. "
+                f"Original error: {exc}"
+            ) from exc
+
+    async def _agenerate(self, prompt):
+        """Wrapper method for easier testing"""
+        return (await self.llm.agenerate(prompt, None)).choices[0].message.content
+
+    async def evaluate_answer(
+        self,
+        question: str,
+        reference_answer: str,
+        actual_answer: str
+    ) -> tuple[int, int, int, str]:
+        if any(not s.strip() for s in [question, reference_answer, actual_answer]):
+            raise ValueError("The question of the reference or the actual answer is a blank "
+                             "string!")
+        prompt = self.prompt_template.format(
+            question=question,
+            reference_answer=reference_answer,
+            actual_answer=actual_answer,
+        )
+        response_str = await self._agenerate(prompt)
+        return self.extract_response_values(response_str)
+
+    async def get_correctness_dict(
+        self,
+        reference: dict,
+        actual: dict,
+    ):
+        result = {"reference_answer": reference["reference_answer"]}
+        try:
+            num_ref_claims, num_actual_claims, num_matching_claims, reason = \
+                await self.evaluate_answer(
+                    reference["question_text"],
+                    reference["reference_answer"],
+                    actual["actual_answer"],
+                )
+            result.update({
+                "answer_reference_claims_count": num_ref_claims,
+                "answer_actual_claims_count": num_actual_claims,
+                "answer_matching_claims_count": num_matching_claims,
+                "answer_correctness_reason": reason,
+            })
+            recall, precision, f1 = self.compute_recall_precision_f1(
+                num_ref_claims, num_actual_claims, num_matching_claims
+            )
+            if recall is not None:
+                result["answer_recall"] = recall
+            if precision is not None:
+                result["answer_precision"] = precision
+            if f1 is not None:
+                result["answer_f1"] = f1
+        except Exception as exc:
+            result["answer_eval_error"] = str(exc)
+        return result
+
+    @staticmethod
+    def compute_recall_precision_f1(
+        n_pos: int,
+        n_pred_pos: int,
+        n_true_pos: int,
+    ) -> tuple[float | None, float | None, float | None]:
+        recall = None
+        precision = None
+        if n_pos:
+            recall = n_true_pos / n_pos
+        if n_pred_pos:
+            precision = n_true_pos / n_pred_pos
+        return recall, precision, compute_f1(recall, precision)
+
+    @staticmethod
+    def extract_response_values(
+        response: str
+    ) -> tuple[int, int, int, str]:
+        vals = response.split("\t")
+        n = len(vals)
+        if n < 4:
+            raise ValueError(f"Expected 4 tab-separated values: {response}")
+        vals = vals[:4]
+        try:
+            n_ref, n_actual, n_matching = map(int, vals[:3])
+        except ValueError:
+            raise ValueError(f"Claims counts should be ints: {vals}")
+        if any([
+            n_ref < 1,
+            n_actual < 1,
+            n_matching < 0,
+            n_matching > n_ref,
+            n_matching > n_actual
+        ]):
+            raise ValueError(
+                f"Invalid claims counts combination: {n_ref}\t{n_actual}\t{n_matching}"
+            )
+        return n_ref, n_actual, n_matching, vals[3]

graphrag_eval-6.3.0/graphrag_eval/cli/answer_correctness.py

@@ -0,0 +1,105 @@
+import argparse
+import asyncio
+import csv
+from argparse import ArgumentParser
+from pathlib import Path
+
+from tqdm import tqdm
+
+from graphrag_eval import llm_factory
+from graphrag_eval.answer_correctness import AnswerCorrectnessEvaluator
+from graphrag_eval.evaluation import Config
+
+
+def parse_args() -> argparse.Namespace:
+    parser = ArgumentParser(
+        description="Calculates answer correctness over the entries from the input tsv file and "
+                    "stores the output in the output tsv file.",
+    )
+    parser.add_argument(
+        "-i",
+        "--input-tsv-file-path",
+        type=Path,
+        required=True,
+        help="Input tsv file path with columns `Question`, `Reference answer` and `Actual answer`",
+    )
+    parser.add_argument(
+        "-o",
+        "--output-tsv-file-path",
+        type=Path,
+        required=True,
+        help="Output tsv file path with columns `#Reference`, `#PTarget`, `#Matching`, "
+             "`Reasoning`, `Error`",
+    )
+    parser.add_argument(
+        "-c",
+        "--config-yaml-file-path",
+        type=Path,
+        required=True,
+        help="Config yaml file path with definition of the LLM to use and optionally a custom "
+             "prompt.",
+    )
+    return parser.parse_args()
+
+
+async def evaluate_and_write(
+    input_tsv_file_path: Path,
+    output_tsv_file_path: Path,
+    evaluator: AnswerCorrectnessEvaluator,
+) -> None:
+    with open(input_tsv_file_path, encoding="utf-8") as f:
+        reader = csv.DictReader(f, delimiter="\t")
+        rows = [row for row in reader]
+    print(f"Writing results to {output_tsv_file_path}")
+    output_tsv_file_path.parent.mkdir(parents=True, exist_ok=True)
+    with open(output_tsv_file_path, "w", encoding="utf-8") as f:
+        writer = csv.writer(f, delimiter="\t")
+        writer.writerow(["#Reference", "#PTarget", "#Matching", "Reasoning", "Error"])
+
+        for row in tqdm(rows):
+            if "Question" not in row or \
+                "Reference answer" not in row or \
+                "Actual answer" not in row:
+                raise ValueError("Unexpected input format!")
+
+            try:
+                vals = await evaluator.evaluate_answer(
+                    row["Question"],
+                    row["Reference answer"],
+                    row["Actual answer"]
+                )
+                vals = vals + ("",)
+                writer.writerow(vals)
+            except Exception as exc:
+                writer.writerow(["", "", "", "", str(exc)])
+            f.flush()
+
+
+def run(
+    config_yaml_file_path: Path,
+    input_tsv_file_path: Path,
+    output_tsv_file_path: Path,
+):
+    config = Config.parse(config_yaml_file_path)
+    ragas_llm = llm_factory.create_llm(config)
+    if ragas_llm is None:
+        raise ValueError("LLM must be configured to calculate the answer correctness!")
+    else:
+        evaluator = AnswerCorrectnessEvaluator(
+            llm=ragas_llm,
+            config=config.answer_correctness,
+        )
+        asyncio.run(evaluate_and_write(
+            input_tsv_file_path,
+            output_tsv_file_path,
+            evaluator,
+        ))
+
+
+def main():
+    args = parse_args()
+    run(
+        args.config_yaml_file_path,
+        args.input_tsv_file_path,
+        args.output_tsv_file_path,
+    )

{graphrag_eval-6.1.0 → graphrag_eval-6.3.0}/graphrag_eval/custom_evaluation.py

@@ -3,6 +3,7 @@ from typing import Literal
 
 from pydantic import BaseModel, ConfigDict, Field, model_validator
 
+from graphrag_eval.llm_factory import create_llm
 
 RESERVED_KEYS = {
     "template_id",
@@ -31,7 +32,6 @@ RESERVED_KEYS = {
     "elapsed_sec",
 }
 
-
 Inputs = Literal[
     "question",
     "reference_answer",
@@ -84,8 +84,8 @@ def create_prompt_template(config: Config, output_variables: list[str]) -> str:
     
     output_variables specifies the order of the outputs.
     """
-    output_instructions = "Output the following values separated by tabs:"\
-        + "".join(f"\n- {k}: {config.outputs[k]}" for k in output_variables)
+    output_instructions = "Output the following values separated by tabs:" \
+                          + "".join(f"\n- {k}: {config.outputs[k]}" for k in output_variables)
     inputs_template = "\n\n".join(
         create_input_template(k) for k in config.inputs
     )
@@ -98,9 +98,9 @@ def create_prompt_template(config: Config, output_variables: list[str]) -> str:
 
 class CustomEvaluator:
     def __init__(
-        self, 
+        self,
         config: Config,
-        llm: "InstructorBaseRagasLLM",
+        eval_config: "evaluation.Config",
     ):
         self.name = config.name
         self.input_variables = config.inputs
@@ -111,11 +111,11 @@ class CustomEvaluator:
             config,
             self.output_variables
         )
-        self.llm = llm
+        self.llm = create_llm(eval_config)
 
-    def _generate(self, prompt: str) -> str:
+    async def _agenerate(self, prompt: str) -> str:
         """Wrapper method for easier testing"""
-        return self.llm.generate(prompt, None).choices[0].message.content
+        return (await self.llm.agenerate(prompt, None)).choices[0].message.content
 
     def format_steps(self, steps: list) -> str:
         steps_formatted = []
@@ -134,9 +134,9 @@ class CustomEvaluator:
                         step_out[k] = val
                 else:
                     step_out[k] = val
-            steps_formatted.append(step_out)        
+            steps_formatted.append(step_out)
         return json.dumps(steps_formatted, indent=2)
-    
+
     def error(self, msg: str) -> dict:
         result = {k: None for k in self.output_variables}
         result[self.name + '_error'] = msg
@@ -157,7 +157,7 @@ class CustomEvaluator:
             return result
         return self.error(f"Expected {n_exp} tab-separated values, got: {response}")
 
-    def evaluate(self, reference: dict, actual: dict) -> dict[str, str | None]:
+    async def evaluate(self, reference: dict, actual: dict) -> dict[str, str | None]:
         inputs = {}
         if "question" in self.input_variables:
             if "question_text" not in reference:
@@ -176,7 +176,7 @@ class CustomEvaluator:
                 return self.error("Reference missing key 'reference_steps'")
             try:
                 formatted_steps_lists = [
-                    self.format_steps(group) 
+                    self.format_steps(group)
                     for group in reference["reference_steps"]
                 ]
             except json.JSONDecodeError:
@@ -191,14 +191,14 @@ class CustomEvaluator:
                 return self.error("Malformed actual step JSON")
             inputs["actual_steps"] = formatted_steps_lists
         prompt = self.prompt_template.format(**inputs)
-        response = self._generate(prompt)
+        response = await self._agenerate(prompt)
         return self.parse_outputs(response)
 
 
 def create_evaluators(config: "evaluation.Config") -> list[CustomEvaluator]:
     if config.custom_evaluations and config.llm:
         return [
-            CustomEvaluator(c, config.llm.generation)
-            for c in config.custom_evaluations
+            CustomEvaluator(custom_evaluation_config, config)
+            for custom_evaluation_config in config.custom_evaluations
         ]
     return []

{graphrag_eval-6.1.0 → graphrag_eval-6.3.0}/graphrag_eval/evaluation.py

@@ -4,7 +4,8 @@ import yaml
 from pydantic import BaseModel, Field, model_validator
 
 from . import custom_evaluation
-from .llm import Config as LLMConfig, create_llm, create_embedder
+from .answer_correctness import AnswerCorrectnessConfig
+from .llm_factory import Config as LLMConfig, create_llm, create_embedder
 from .steps.evaluation import evaluate_steps
 
 
@@ -12,6 +13,7 @@ class Config(BaseModel):
     llm: LLMConfig | None = None
     custom_evaluations: list[custom_evaluation.Config] | None \
         = Field(default=None, min_length=1)
+    answer_correctness: AnswerCorrectnessConfig | None = None
 
     @model_validator(mode="after")
     def validate_config(self) -> "Config":
@@ -19,7 +21,7 @@ class Config(BaseModel):
             msg = "llm config is required if custom_evaluations are provided"
             raise ValueError(msg)
         return self
-    
+
     @classmethod
     def parse(cls, config_file_path: str | Path | None) -> "Config":
         if config_file_path:
@@ -75,10 +77,11 @@ async def run_evaluation(
                 if "reference_answer" in question and ragas_llm:
                     from graphrag_eval.answer_correctness import AnswerCorrectnessEvaluator
                     answer_correctness_evaluator = AnswerCorrectnessEvaluator(
-                        llm=ragas_llm
+                        llm=ragas_llm,
+                        config=config.answer_correctness,
                     )
                     eval_result.update(
-                        answer_correctness_evaluator.get_correctness_dict(
+                        await answer_correctness_evaluator.get_correctness_dict(
                             question,
                             actual_result,
                         )
@@ -90,8 +93,8 @@ async def run_evaluation(
                     ragas_llm,
                 )
             )
-            for relevance_evaluator in custom_evaluators:
-                custom_metrics = relevance_evaluator.evaluate(question, actual_result)
+            for custom_evaluator in custom_evaluators:
+                custom_metrics = await custom_evaluator.evaluate(question, actual_result)
                 eval_result.update(**custom_metrics)
             for key in "input_tokens", "output_tokens", "total_tokens", "elapsed_sec":
                 if key in actual_result:

graphrag_eval-6.1.0/graphrag_eval/llm.py → graphrag_eval-6.3.0/graphrag_eval/llm_factory.py

@@ -6,8 +6,8 @@ from pydantic import BaseModel, ConfigDict, Field
 class GenerationConfig(BaseModel):
     provider: str
     model: str
-    temperature: float = Field(ge=0.0, le=2.0)
-    max_tokens: int = Field(ge=1)
+    temperature: float = Field(default=0.0, ge=0.0, le=2.0)
+    max_tokens: int | None = Field(default=None, ge=1)
     model_config = ConfigDict(extra='allow')
 
 
@@ -37,6 +37,7 @@ def create_llm(config: "evaluation.Config") -> Optional["InstructorBaseRagasLLM"
         )
         ragas_llm.is_async = True
         return ragas_llm
+    return None
 
 
 def create_embedder(config: "evaluation.Config") -> Optional["BaseRagasEmbedding"]:
@@ -53,3 +54,4 @@ def create_embedder(config: "evaluation.Config") -> Optional["BaseRagasEmbedding
             **params,
         )
         return ragas_embedder
+    return None

{graphrag_eval-6.1.0 → graphrag_eval-6.3.0}/graphrag_eval/prompts/template.md

@@ -10,7 +10,7 @@ Below are a query, a reference response and a candidate response to it.
 {reference_answer}
 
 # Candidate response
-{candidate_answer}
+{actual_answer}
 
 # Output values
 * v1: Count of reference response claims

{graphrag_eval-6.1.0 → graphrag_eval-6.3.0}/pyproject.toml

@@ -1,6 +1,6 @@
 [project]
 name = "graphrag-eval"
-version = "6.1.0"
+version = "6.3.0"
 description = "For assessing question answering systems' final answers and intermediate steps, against a given set of questions, reference answers and steps."
 authors = [
     { name = "Philip Ganchev", email = "philip.ganchev@graphwise.ai" },
@@ -24,7 +24,7 @@ llm = ["ragas", "litellm", "pyyaml"]
 
 [tool.poetry.group.llm.dependencies]
 ragas = "0.4.3"
-litellm = "1.83.14"
+litellm = "1.85.1"
 pyyaml = "6.0.3"
 
 [tool.poetry.group.llm]
@@ -41,7 +41,7 @@ pyyaml = "6.0.3"
 optional = true
 
 [project.scripts]
-answer-correctness = "graphrag_eval.answer_correctness:main"
+answer-correctness = "graphrag_eval.cli.answer_correctness:main"
 
 [build-system]
 requires = ["poetry-core>=2.0.0"]