graphrag-eval 6.2.0__tar.gz → 6.3.0__tar.gz

graphrag_eval-6.3.0/PKG-INFO

@@ -0,0 +1,46 @@
+Metadata-Version: 2.3
+Name: graphrag-eval
+Version: 6.3.0
+Summary: For assessing question answering systems' final answers and intermediate steps, against a given set of questions, reference answers and steps.
+License: Apache-2.0
+Author: Philip Ganchev
+Author-email: philip.ganchev@graphwise.ai
+Requires-Python: >=3.12,<3.13
+Classifier: License :: OSI Approved :: Apache Software License
+Classifier: Programming Language :: Python :: 3
+Classifier: Programming Language :: Python :: 3.12
+Provides-Extra: llm
+Requires-Dist: pydantic (==2.12.5)
+Requires-Dist: python-dateutil (==2.9.0.post0)
+Requires-Dist: ragas (==0.4.3) ; extra == "llm"
+Project-URL: Repository, https://github.com/Ontotext-AD/graphrag-eval
+Description-Content-Type: text/markdown
+
+<p align="center">
+  <img alt="Graphwise Logo" src="https://github.com/Ontotext-AD/graphrag-eval/blob/main/.github/Graphwise_Logo.jpg">
+</p>
+
+# QA Evaluation
+
+This is a Python library for assessing the quality of question-answering systems, such as systems built with LLM-based agents. It is agnostic to the agent implementation and the LLM it uses.
+
+The evaluation is based on a user-provided reference dataset containing queries, reference responses, and optional reference steps, such as expected tool uses. The evaluator compares these references with the agent's actual responses and executed steps. Reference steps can be grouped to allow some expected steps to occur in any order.
+
+The library provides built-in evaluation metrics and supports user-defined custom metrics ([§ Metrics](https://github.com/Ontotext-AD/graphrag-eval/blob/main/docs/metrics.md)).
+
+## Documentation
+
+- [Quickstart](https://github.com/Ontotext-AD/graphrag-eval/blob/main/docs/quickstart.md)
+- [Metrics](https://github.com/Ontotext-AD/graphrag-eval/blob/main/docs/metrics.md)
+- [Configuration](https://github.com/Ontotext-AD/graphrag-eval/blob/main/docs/config.md)
+- [Input](https://github.com/Ontotext-AD/graphrag-eval/blob/main/docs/input.md)
+- [Output](https://github.com/Ontotext-AD/graphrag-eval/blob/main/docs/output.md)
+
+## Maintainers
+
+Developed and maintained by [Graphwise](https://graphwise.ai/). For issues and feature requests, please open a [GitHub issue](https://github.com/Ontotext-AD/graphrag-eval/issues).
+
+## License
+
+Apache-2.0 License. See the [LICENSE](https://github.com/Ontotext-AD/graphrag-eval/blob/main/LICENSE) file for details.
+

graphrag_eval-6.3.0/README.md

@@ -0,0 +1,27 @@
+<p align="center">
+  <img alt="Graphwise Logo" src="https://github.com/Ontotext-AD/graphrag-eval/blob/main/.github/Graphwise_Logo.jpg">
+</p>
+
+# QA Evaluation
+
+This is a Python library for assessing the quality of question-answering systems, such as systems built with LLM-based agents. It is agnostic to the agent implementation and the LLM it uses.
+
+The evaluation is based on a user-provided reference dataset containing queries, reference responses, and optional reference steps, such as expected tool uses. The evaluator compares these references with the agent's actual responses and executed steps. Reference steps can be grouped to allow some expected steps to occur in any order.
+
+The library provides built-in evaluation metrics and supports user-defined custom metrics ([§ Metrics](https://github.com/Ontotext-AD/graphrag-eval/blob/main/docs/metrics.md)).
+
+## Documentation
+
+- [Quickstart](https://github.com/Ontotext-AD/graphrag-eval/blob/main/docs/quickstart.md)
+- [Metrics](https://github.com/Ontotext-AD/graphrag-eval/blob/main/docs/metrics.md)
+- [Configuration](https://github.com/Ontotext-AD/graphrag-eval/blob/main/docs/config.md)
+- [Input](https://github.com/Ontotext-AD/graphrag-eval/blob/main/docs/input.md)
+- [Output](https://github.com/Ontotext-AD/graphrag-eval/blob/main/docs/output.md)
+
+## Maintainers
+
+Developed and maintained by [Graphwise](https://graphwise.ai/). For issues and feature requests, please open a [GitHub issue](https://github.com/Ontotext-AD/graphrag-eval/issues).
+
+## License
+
+Apache-2.0 License. See the [LICENSE](https://github.com/Ontotext-AD/graphrag-eval/blob/main/LICENSE) file for details.

graphrag_eval-6.3.0/graphrag_eval/answer_correctness.py

@@ -0,0 +1,139 @@
+from pathlib import Path
+
+from pydantic import BaseModel, Field
+
+from graphrag_eval.util import compute_f1
+
+
+def load_default_prompt() -> str:
+    with open(Path(__file__).parent / "prompts" / "template.md", "r", encoding="utf-8") as f:
+        return f.read()
+
+
+class AnswerCorrectnessConfig(BaseModel):
+    prompt: str = Field(default_factory=load_default_prompt)
+
+
+class InvalidPromptException(Exception):
+    def __init__(self, message="The prompt template is invalid and cannot be formatted."):
+        self.message = message
+        super().__init__(self.message)
+
+
+class AnswerCorrectnessEvaluator:
+    def __init__(
+        self,
+        llm: "InstructorBaseRagasLLM",
+        config: AnswerCorrectnessConfig | None = None,
+    ):
+        self.config = config or AnswerCorrectnessConfig()
+        self.__validate_prompt_template(self.config.prompt)
+        self.prompt_template = self.config.prompt
+        self.llm = llm
+
+    @staticmethod
+    def __validate_prompt_template(prompt_template: str):
+        try:
+            prompt_template.format(
+                question="Q?",
+                reference_answer="R",
+                actual_answer="A",
+            )
+        except Exception as exc:
+            raise InvalidPromptException(
+                "Invalid prompt template. Must only contain placeholders: "
+                "{question}, {reference_answer}, and {actual_answer}. "
+                f"Original error: {exc}"
+            ) from exc
+
+    async def _agenerate(self, prompt):
+        """Wrapper method for easier testing"""
+        return (await self.llm.agenerate(prompt, None)).choices[0].message.content
+
+    async def evaluate_answer(
+        self,
+        question: str,
+        reference_answer: str,
+        actual_answer: str
+    ) -> tuple[int, int, int, str]:
+        if any(not s.strip() for s in [question, reference_answer, actual_answer]):
+            raise ValueError("The question of the reference or the actual answer is a blank "
+                             "string!")
+        prompt = self.prompt_template.format(
+            question=question,
+            reference_answer=reference_answer,
+            actual_answer=actual_answer,
+        )
+        response_str = await self._agenerate(prompt)
+        return self.extract_response_values(response_str)
+
+    async def get_correctness_dict(
+        self,
+        reference: dict,
+        actual: dict,
+    ):
+        result = {"reference_answer": reference["reference_answer"]}
+        try:
+            num_ref_claims, num_actual_claims, num_matching_claims, reason = \
+                await self.evaluate_answer(
+                    reference["question_text"],
+                    reference["reference_answer"],
+                    actual["actual_answer"],
+                )
+            result.update({
+                "answer_reference_claims_count": num_ref_claims,
+                "answer_actual_claims_count": num_actual_claims,
+                "answer_matching_claims_count": num_matching_claims,
+                "answer_correctness_reason": reason,
+            })
+            recall, precision, f1 = self.compute_recall_precision_f1(
+                num_ref_claims, num_actual_claims, num_matching_claims
+            )
+            if recall is not None:
+                result["answer_recall"] = recall
+            if precision is not None:
+                result["answer_precision"] = precision
+            if f1 is not None:
+                result["answer_f1"] = f1
+        except Exception as exc:
+            result["answer_eval_error"] = str(exc)
+        return result
+
+    @staticmethod
+    def compute_recall_precision_f1(
+        n_pos: int,
+        n_pred_pos: int,
+        n_true_pos: int,
+    ) -> tuple[float | None, float | None, float | None]:
+        recall = None
+        precision = None
+        if n_pos:
+            recall = n_true_pos / n_pos
+        if n_pred_pos:
+            precision = n_true_pos / n_pred_pos
+        return recall, precision, compute_f1(recall, precision)
+
+    @staticmethod
+    def extract_response_values(
+        response: str
+    ) -> tuple[int, int, int, str]:
+        vals = response.split("\t")
+        n = len(vals)
+        if n < 4:
+            raise ValueError(f"Expected 4 tab-separated values: {response}")
+        vals = vals[:4]
+        try:
+            n_ref, n_actual, n_matching = map(int, vals[:3])
+        except ValueError:
+            raise ValueError(f"Claims counts should be ints: {vals}")
+        if any([
+            n_ref < 1,
+            n_actual < 1,
+            n_matching < 0,
+            n_matching > n_ref,
+            n_matching > n_actual
+        ]):
+            raise ValueError(
+                f"Invalid claims counts combination: {n_ref}\t{n_actual}\t{n_matching}"
+            )
+        return n_ref, n_actual, n_matching, vals[3]

graphrag_eval-6.3.0/graphrag_eval/cli/answer_correctness.py

@@ -0,0 +1,105 @@
+import argparse
+import asyncio
+import csv
+from argparse import ArgumentParser
+from pathlib import Path
+
+from tqdm import tqdm
+
+from graphrag_eval import llm_factory
+from graphrag_eval.answer_correctness import AnswerCorrectnessEvaluator
+from graphrag_eval.evaluation import Config
+
+
+def parse_args() -> argparse.Namespace:
+    parser = ArgumentParser(
+        description="Calculates answer correctness over the entries from the input tsv file and "
+                    "stores the output in the output tsv file.",
+    )
+    parser.add_argument(
+        "-i",
+        "--input-tsv-file-path",
+        type=Path,
+        required=True,
+        help="Input tsv file path with columns `Question`, `Reference answer` and `Actual answer`",
+    )
+    parser.add_argument(
+        "-o",
+        "--output-tsv-file-path",
+        type=Path,
+        required=True,
+        help="Output tsv file path with columns `#Reference`, `#PTarget`, `#Matching`, "
+             "`Reasoning`, `Error`",
+    )
+    parser.add_argument(
+        "-c",
+        "--config-yaml-file-path",
+        type=Path,
+        required=True,
+        help="Config yaml file path with definition of the LLM to use and optionally a custom "
+             "prompt.",
+    )
+    return parser.parse_args()
+
+
+async def evaluate_and_write(
+    input_tsv_file_path: Path,
+    output_tsv_file_path: Path,
+    evaluator: AnswerCorrectnessEvaluator,
+) -> None:
+    with open(input_tsv_file_path, encoding="utf-8") as f:
+        reader = csv.DictReader(f, delimiter="\t")
+        rows = [row for row in reader]
+    print(f"Writing results to {output_tsv_file_path}")
+    output_tsv_file_path.parent.mkdir(parents=True, exist_ok=True)
+    with open(output_tsv_file_path, "w", encoding="utf-8") as f:
+        writer = csv.writer(f, delimiter="\t")
+        writer.writerow(["#Reference", "#PTarget", "#Matching", "Reasoning", "Error"])
+
+        for row in tqdm(rows):
+            if "Question" not in row or \
+                "Reference answer" not in row or \
+                "Actual answer" not in row:
+                raise ValueError("Unexpected input format!")
+
+            try:
+                vals = await evaluator.evaluate_answer(
+                    row["Question"],
+                    row["Reference answer"],
+                    row["Actual answer"]
+                )
+                vals = vals + ("",)
+                writer.writerow(vals)
+            except Exception as exc:
+                writer.writerow(["", "", "", "", str(exc)])
+            f.flush()
+
+
+def run(
+    config_yaml_file_path: Path,
+    input_tsv_file_path: Path,
+    output_tsv_file_path: Path,
+):
+    config = Config.parse(config_yaml_file_path)
+    ragas_llm = llm_factory.create_llm(config)
+    if ragas_llm is None:
+        raise ValueError("LLM must be configured to calculate the answer correctness!")
+    else:
+        evaluator = AnswerCorrectnessEvaluator(
+            llm=ragas_llm,
+            config=config.answer_correctness,
+        )
+        asyncio.run(evaluate_and_write(
+            input_tsv_file_path,
+            output_tsv_file_path,
+            evaluator,
+        ))
+
+
+def main():
+    args = parse_args()
+    run(
+        args.config_yaml_file_path,
+        args.input_tsv_file_path,
+        args.output_tsv_file_path,
+    )

{graphrag_eval-6.2.0 → graphrag_eval-6.3.0}/graphrag_eval/evaluation.py

@@ -4,6 +4,7 @@ import yaml
 from pydantic import BaseModel, Field, model_validator
 
 from . import custom_evaluation
+from .answer_correctness import AnswerCorrectnessConfig
 from .llm_factory import Config as LLMConfig, create_llm, create_embedder
 from .steps.evaluation import evaluate_steps
 
@@ -12,6 +13,7 @@ class Config(BaseModel):
     llm: LLMConfig | None = None
     custom_evaluations: list[custom_evaluation.Config] | None \
         = Field(default=None, min_length=1)
+    answer_correctness: AnswerCorrectnessConfig | None = None
 
     @model_validator(mode="after")
     def validate_config(self) -> "Config":
@@ -75,7 +77,8 @@ async def run_evaluation(
                 if "reference_answer" in question and ragas_llm:
                     from graphrag_eval.answer_correctness import AnswerCorrectnessEvaluator
                     answer_correctness_evaluator = AnswerCorrectnessEvaluator(
-                        llm=ragas_llm
+                        llm=ragas_llm,
+                        config=config.answer_correctness,
                     )
                     eval_result.update(
                         await answer_correctness_evaluator.get_correctness_dict(

{graphrag_eval-6.2.0 → graphrag_eval-6.3.0}/graphrag_eval/llm_factory.py

@@ -6,8 +6,8 @@ from pydantic import BaseModel, ConfigDict, Field
 class GenerationConfig(BaseModel):
     provider: str
     model: str
-    temperature: float = Field(ge=0.0, le=2.0)
-    max_tokens: int = Field(ge=1)
+    temperature: float = Field(default=0.0, ge=0.0, le=2.0)
+    max_tokens: int | None = Field(default=None, ge=1)
     model_config = ConfigDict(extra='allow')
 
 

{graphrag_eval-6.2.0 → graphrag_eval-6.3.0}/graphrag_eval/prompts/template.md

@@ -10,7 +10,7 @@ Below are a query, a reference response and a candidate response to it.
 {reference_answer}
 
 # Candidate response
-{candidate_answer}
+{actual_answer}
 
 # Output values
 * v1: Count of reference response claims

{graphrag_eval-6.2.0 → graphrag_eval-6.3.0}/pyproject.toml

@@ -1,6 +1,6 @@
 [project]
 name = "graphrag-eval"
-version = "6.2.0"
+version = "6.3.0"
 description = "For assessing question answering systems' final answers and intermediate steps, against a given set of questions, reference answers and steps."
 authors = [
     { name = "Philip Ganchev", email = "philip.ganchev@graphwise.ai" },
@@ -41,7 +41,7 @@ pyyaml = "6.0.3"
 optional = true
 
 [project.scripts]
-answer-correctness = "graphrag_eval.answer_correctness:main"
+answer-correctness = "graphrag_eval.cli.answer_correctness:main"
 
 [build-system]
 requires = ["poetry-core>=2.0.0"]