graphrag-eval 5.0.1__tar.gz → 5.0.2__tar.gz

{graphrag_eval-5.0.1 → graphrag_eval-5.0.2}/PKG-INFO

@@ -1,10 +1,10 @@
 Metadata-Version: 2.3
 Name: graphrag-eval
-Version: 5.0.1
+Version: 5.0.2
 Summary: For assessing question answering systems' final answers and intermediate steps, against a given set of questions, reference answers and steps.
 License: Apache-2.0
-Author: Neli Hateva
-Author-email: neli.hateva@graphwise.ai
+Author: Philip Ganchev
+Author-email: philip.ganchev@graphwise.ai
 Requires-Python: >=3.12,<3.13
 Classifier: License :: OSI Approved :: Apache Software License
 Classifier: Programming Language :: Python :: 3
@@ -74,7 +74,7 @@ To evaluate answers and/or steps:
 1. Format the answers and/or steps you want to evaluate: section [Evaluation Target Corpus](#Evaluation-Target-Corpus)
 1. To evaluate answer relevance:
     1. Include `actual_answer` in the target data to evaluate
-    1. Set environment vairabe `OPENAI_API_KEY` appropriately
+    1. Set environment variable `OPENAI_API_KEY` appropriately
 1. To evaluate answer correctness:
     1. Include `reference_answer` in the reference corpus and `actual_answer` in the target data to evaluate
     1. Set environment variable `OPENAI_API_KEY` appropriately

{graphrag_eval-5.0.1 → graphrag_eval-5.0.2}/README.md

@@ -56,7 +56,7 @@ To evaluate answers and/or steps:
 1. Format the answers and/or steps you want to evaluate: section [Evaluation Target Corpus](#Evaluation-Target-Corpus)
 1. To evaluate answer relevance:
     1. Include `actual_answer` in the target data to evaluate
-    1. Set environment vairabe `OPENAI_API_KEY` appropriately
+    1. Set environment variable `OPENAI_API_KEY` appropriately
 1. To evaluate answer correctness:
     1. Include `reference_answer` in the reference corpus and `actual_answer` in the target data to evaluate
     1. Set environment variable `OPENAI_API_KEY` appropriately

{graphrag_eval-5.0.1 → graphrag_eval-5.0.2}/graphrag_eval/answer_correctness.py

@@ -6,7 +6,7 @@ from tqdm import tqdm
 
 
 IN_FILE_PATH = "../data/data-1.tsv"
-PROMPT_FILE_PATH = "prompts/template.md"
+PROMPT_FILE_PATH = Path(__file__).parent / "prompts" / "template.md"
 OUT_FILE_PATH = "results/data-1.tsv"
 OUT_FIELDS = ["#Reference", "#PTarget", "#Matching", "Reasoning", "Error"]
 LLM_MODEL = "gpt-4o-mini"

graphrag_eval-5.0.2/graphrag_eval/prompts/template.md

@@ -0,0 +1,26 @@
+Below are a query, a reference response and a candidate response to it.
+1. Extract all claims from each response
+2. Find matching claims between responses. Matching claims have the same meaning and details such as subjects, names, locations, amounts, IDs, commands and paths.
+3. Output the values listed below (all and only those).
+
+# Query
+{question}
+
+# Reference response
+{reference_answer}
+
+# Candidate response
+{candidate_answer}
+
+# Output values
+* v1: Count of reference response claims
+* v2: Count of candidate response claims
+* v3: Count of matching claims
+* v4: Explanation of v1-v3 in English
+
+# Value checks
+* 1 <= v1, v2
+* 0 <= v3 <= v1, v2
+
+# Output format
+<v1><tab><v2><tab><v3><tab><v4>

{graphrag_eval-5.0.1 → graphrag_eval-5.0.2}/pyproject.toml

@@ -1,10 +1,11 @@
 [project]
 name = "graphrag-eval"
-version = "5.0.1"
+version = "5.0.2"
 description = "For assessing question answering systems' final answers and intermediate steps, against a given set of questions, reference answers and steps."
 authors = [
-    {name = "Neli Hateva", email = "neli.hateva@graphwise.ai"},
-    {name = "Philip Ganchev", email = "philip.ganchev@graphwise.ai"}
+  { name = "Philip Ganchev", email = "philip.ganchev@graphwise.ai" },
+  { name = "Aleksis Datseris", email = "aleksis.datseris@graphwise.ai" },
+  { name = "Neli Hateva", email = "neli.hateva@graphwise.ai" },
 ]
 readme = "README.md"
 license = "Apache-2.0"