graphrag-eval 6.4.0__tar.gz → 6.5.0__tar.gz

{graphrag_eval-6.4.0 → graphrag_eval-6.5.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.3
 Name: graphrag-eval
-Version: 6.4.0
+Version: 6.5.0
 Summary: For assessing question answering systems' final answers and intermediate steps, against a given set of questions, reference answers and steps.
 License: Apache-2.0
 Author: Philip Ganchev
@@ -10,7 +10,7 @@ Classifier: License :: OSI Approved :: Apache Software License
 Classifier: Programming Language :: Python :: 3
 Classifier: Programming Language :: Python :: 3.12
 Provides-Extra: llm
-Requires-Dist: pydantic (==2.12.5)
+Requires-Dist: pydantic (==2.14.1)
 Requires-Dist: python-dateutil (==2.9.0.post0)
 Requires-Dist: ragas (==0.4.3) ; extra == "llm"
 Project-URL: Repository, https://github.com/Ontotext-AD/graphrag-eval

{graphrag_eval-6.4.0 → graphrag_eval-6.5.0}/graphrag_eval/answer_correctness.py

@@ -3,7 +3,7 @@ from __future__ import annotations
 from pathlib import Path
 from typing import Any, Self, TYPE_CHECKING
 
-from pydantic import BaseModel, Field
+from pydantic import BaseModel, ConfigDict, Field, model_validator
 
 from graphrag_eval.util import compute_f1
 from .evaluator import Evaluator
@@ -25,6 +25,27 @@ class AnswerCorrectnessConfig(BaseModel):
     prompt: str = Field(default_factory=load_default_prompt)
 
 
+class AnswerCorrectnessResult(BaseModel):
+    model_config = ConfigDict(extra="forbid")
+
+    reference_claims_count: int = Field(ge=1)
+    actual_claims_count: int = Field(ge=1)
+    matching_claims_count: int = Field(ge=0)
+    reason: str = Field(min_length=1)
+
+    @model_validator(mode="after")
+    def validate_claim_counts(self) -> "AnswerCorrectnessResult":
+        if (
+            self.matching_claims_count > self.reference_claims_count or
+            self.matching_claims_count > self.actual_claims_count
+        ):
+            raise ValueError(
+                "matching_claims_count must be less than or equal to both "
+                "reference_claims_count and actual_claims_count"
+            )
+        return self
+
+
 class InvalidPromptException(Exception):
     def __init__(
         self,
@@ -73,16 +94,17 @@ class AnswerCorrectnessEvaluator:
                 f"Original error: {exc}"
             ) from exc
 
-    async def _agenerate(self, prompt):
+    async def _agenerate(self, prompt: str) -> AnswerCorrectnessResult:
         """Wrapper method for easier testing"""
-        return (await self.ragas_llm.agenerate(prompt, None)).choices[0].message.content
+        response = await self.ragas_llm.agenerate(prompt, AnswerCorrectnessResult)
+        return AnswerCorrectnessResult.model_validate(response)
 
     async def evaluate_answer(
         self,
         question: str,
         reference_answer: str,
         actual_answer: str
-    ) -> tuple[int, int, int, str]:
+    ) -> AnswerCorrectnessResult:
         if any(
             not s.strip() for s in [question, reference_answer, actual_answer]
         ):
@@ -95,8 +117,7 @@ class AnswerCorrectnessEvaluator:
             reference_answer=reference_answer,
             actual_answer=actual_answer,
         )
-        response_str = await self._agenerate(prompt)
-        return self.extract_response_values(response_str)
+        return await self._agenerate(prompt)
 
     async def evaluate(
         self,
@@ -107,20 +128,21 @@ class AnswerCorrectnessEvaluator:
             return {}
         result = {}
         try:
-            num_ref_claims, num_actual_claims, num_matching_claims, reason = \
-                await self.evaluate_answer(
-                    reference["question_text"],
-                    reference["reference_answer"],
-                    actual["actual_answer"],
-                )
+            correctness = await self.evaluate_answer(
+                reference["question_text"],
+                reference["reference_answer"],
+                actual["actual_answer"],
+            )
             result.update({
-                "answer_reference_claims_count": num_ref_claims,
-                "answer_actual_claims_count": num_actual_claims,
-                "answer_matching_claims_count": num_matching_claims,
-                "answer_correctness_reason": reason,
+                "answer_reference_claims_count": correctness.reference_claims_count,
+                "answer_actual_claims_count": correctness.actual_claims_count,
+                "answer_matching_claims_count": correctness.matching_claims_count,
+                "answer_correctness_reason": correctness.reason,
             })
             recall, precision, f1 = self.compute_recall_precision_f1(
-                num_ref_claims, num_actual_claims, num_matching_claims
+                correctness.reference_claims_count,
+                correctness.actual_claims_count,
+                correctness.matching_claims_count,
             )
             if recall is not None:
                 result["answer_recall"] = recall
@@ -146,31 +168,5 @@ class AnswerCorrectnessEvaluator:
             precision = n_true_pos / n_pred_pos
         return recall, precision, compute_f1(recall, precision)
 
-    @staticmethod
-    def extract_response_values(
-        response: str
-    ) -> tuple[int, int, int, str]:
-        vals = response.split("\t")
-        n = len(vals)
-        if n < 4:
-            raise ValueError(f"Expected 4 tab-separated values: {response}")
-        vals = vals[:4]
-        try:
-            n_ref, n_actual, n_matching = map(int, vals[:3])
-        except ValueError:
-            raise ValueError(f"Claims counts should be ints: {vals}")
-        if any([
-            n_ref < 1,
-            n_actual < 1,
-            n_matching < 0,
-            n_matching > n_ref,
-            n_matching > n_actual
-        ]):
-            raise ValueError(
-                "Invalid claims counts combination: "
-                f"{n_ref}\t{n_actual}\t{n_matching}"
-            )
-        return n_ref, n_actual, n_matching, vals[3]
-
 
 _: Evaluator = AnswerCorrectnessEvaluator

{graphrag_eval-6.4.0 → graphrag_eval-6.5.0}/graphrag_eval/cli/answer_correctness.py

@@ -3,6 +3,7 @@ from __future__ import annotations
 import argparse
 import asyncio
 import csv
+import sys
 from argparse import ArgumentParser
 from pathlib import Path
 from typing import TYPE_CHECKING
@@ -55,6 +56,8 @@ async def evaluate_and_write(
     output_tsv_file_path: Path,
     evaluator: AnswerCorrectnessEvaluator,
 ) -> None:
+    csv.field_size_limit(sys.maxsize)
+
     with open(input_tsv_file_path, encoding="utf-8") as f:
         reader = csv.DictReader(f, delimiter="\t")
         rows = [row for row in reader]
@@ -73,13 +76,18 @@ async def evaluate_and_write(
                 raise ValueError("Unexpected input format!")
 
             try:
-                vals = await evaluator.evaluate_answer(
+                correctness = await evaluator.evaluate_answer(
                     row["Question"],
                     row["Reference answer"],
                     row["Actual answer"]
                 )
-                vals = vals + ("",)
-                writer.writerow(vals)
+                writer.writerow([
+                    correctness.reference_claims_count,
+                    correctness.actual_claims_count,
+                    correctness.matching_claims_count,
+                    correctness.reason,
+                    "",
+                ])
             except Exception as exc:
                 writer.writerow(["", "", "", "", str(exc)])
             f.flush()

graphrag_eval-6.5.0/graphrag_eval/prompts/template.md

@@ -0,0 +1,31 @@
+Below are a query, a reference response and a candidate response to it.
+1. Extract all claims from each response.
+2. Find matching claims between responses. Matching claims have the same meaning and details such as subjects, names, locations, amounts, IDs, commands and paths.
+3. Output one JSON object only. Do not include Markdown, code fences, or any text outside the JSON object.
+
+# Query
+{question}
+
+# Reference response
+{reference_answer}
+
+# Candidate response
+{actual_answer}
+
+# JSON fields
+* reference_claims_count: Count of reference response claims
+* actual_claims_count: Count of candidate response claims
+* matching_claims_count: Count of matching claims
+* reason: Explanation of the counts in English
+
+# Value checks
+* 1 <= reference_claims_count, actual_claims_count
+* 0 <= matching_claims_count <= reference_claims_count, actual_claims_count
+
+# Output format
+{{
+  "reference_claims_count": 1,
+  "actual_claims_count": 1,
+  "matching_claims_count": 1,
+  "reason": "The reference and candidate responses contain the same single claim."
+}}

{graphrag_eval-6.4.0 → graphrag_eval-6.5.0}/pyproject.toml

@@ -1,6 +1,6 @@
 [project]
 name = "graphrag-eval"
-version = "6.4.0"
+version = "6.5.0"
 description = "For assessing question answering systems' final answers and intermediate steps, against a given set of questions, reference answers and steps."
 authors = [
     { name = "Philip Ganchev", email = "philip.ganchev@graphwise.ai" },
@@ -17,23 +17,23 @@ repository = "https://github.com/Ontotext-AD/graphrag-eval"
 [tool.poetry.dependencies]
 python-dateutil = "2.9.0.post0"
 ragas = { version = "0.4.3", optional = true }
-pydantic = "2.12.5"
+pydantic = "2.14.1"
 
 [tool.poetry.extras]
 llm = ["ragas", "litellm", "pyyaml"]
 
 [tool.poetry.group.llm.dependencies]
 ragas = "0.4.3"
-litellm = "1.85.1"
+litellm = "1.105.1"
 pyyaml = "6.0.3"
 
 [tool.poetry.group.llm]
 optional = true
 
 [tool.poetry.group.test.dependencies]
-pytest = "<10,>=9.0.3"
-pytest-asyncio = "1.3.0"
-pytest-cov = "<8,>=7"
+pytest = "<10,>=9.1.1"
+pytest-asyncio = "<2,>=1.4.0"
+pytest-cov = "<8,>=7.1.0"
 jsonlines = "4.0.0"
 pyyaml = "6.0.3"
 

graphrag_eval-6.4.0/graphrag_eval/prompts/template.md

@@ -1,26 +0,0 @@
-Below are a query, a reference response and a candidate response to it.
-1. Extract all claims from each response
-2. Find matching claims between responses. Matching claims have the same meaning and details such as subjects, names, locations, amounts, IDs, commands and paths.
-3. Output the values listed below (all and only those).
-
-# Query
-{question}
-
-# Reference response
-{reference_answer}
-
-# Candidate response
-{actual_answer}
-
-# Output values
-* v1: Count of reference response claims
-* v2: Count of candidate response claims
-* v3: Count of matching claims
-* v4: Explanation of v1-v3 in English
-
-# Value checks
-* 1 <= v1, v2
-* 0 <= v3 <= v1, v2
-
-# Output format
-<v1><tab><v2><tab><v3><tab><v4>