aidial-rag-eval 0.6.0.dev15__tar.gz → 0.6.0.dev16__tar.gz

{aidial_rag_eval-0.6.0.dev15 → aidial_rag_eval-0.6.0.dev16}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: aidial-rag-eval
-Version: 0.6.0.dev15
+Version: 0.6.0.dev16
 Summary: Library for RAG evaluation
 License: Apache-2.0
 License-File: LICENSE

{aidial_rag_eval-0.6.0.dev15 → aidial_rag_eval-0.6.0.dev16}/pyproject.toml

@@ -1,6 +1,6 @@
 [tool.poetry]
 name = "aidial-rag-eval"
-version = "0.6.0.dev15"
+version = "0.6.0.dev16"
 description = "Library for RAG evaluation"
 authors = ["EPAM RAIL <SpecialEPM-DIALDevTeam@epam.com>"]
 homepage = "https://github.com/epam/ai-dial-rag-eval"

{aidial_rag_eval-0.6.0.dev15 → aidial_rag_eval-0.6.0.dev16}/src/aidial_rag_eval/generation/inference.py

@@ -440,6 +440,42 @@ def infer_statements(
     )
 
 
+def _aggregate_segment_inferences(
+    grouped_data_item: List[Tuple[InferenceInputs, InferenceScore]],
+) -> Tuple[float, float, float]:
+    """Returns (inference, min_possible_inference, max_possible_inference).
+
+    Four cases:
+    1. No errors, statements present: inference = weighted mean of segment inferences,
+       weighted by statement count.
+    2. No errors, 0 statements total: inference=0.0, min=0.0, max=0.0.
+    3. Pre-inference failure — decontextualization/statement extraction error:
+       inference=nan, min=0.0, max=1.0.
+    4. Inference-stage errors only: inference=nan, min/max derived by substituting 0/1 for
+       failed segments.
+    """
+    if any(inputs.error is not None for inputs, _ in grouped_data_item):
+        return math.nan, 0.0, 1.0
+    total_statements = sum(len(inputs.statements) for inputs, _ in grouped_data_item)
+    if total_statements == 0:
+        return 0.0, 0.0, 0.0
+    inferences = np.array(
+        [score.inference for _, score in grouped_data_item], dtype=float
+    )
+    weights = np.array(
+        [float(len(inputs.statements)) for inputs, _ in grouped_data_item],
+        dtype=float,
+    )
+    inference = float(np.average(inferences, weights=weights))
+    min_possible = float(
+        np.average(np.nan_to_num(inferences, nan=0.0), weights=weights)
+    )
+    max_possible = float(
+        np.average(np.nan_to_num(inferences, nan=1.0), weights=weights)
+    )
+    return inference, min_possible, max_possible
+
+
 def calculate_batch_inference(
     premises: List[Premise],
     hypotheses: List[Hypothesis],
@@ -518,24 +554,14 @@ def calculate_batch_inference(
     inference_returns: List[InferenceReturn] = []
     for hypothesis_index, grouped_data_item in enumerate(grouped_data_list):
         segmented_text = segmented_hypotheses[hypothesis_index]
-        inferences = [score.inference for _, score in grouped_data_item]
+        assert not isinstance(segmented_text, ErrorInfo)
+        mean_inference, min_possible_inference, max_possible_inference = (
+            _aggregate_segment_inferences(grouped_data_item)
+        )
         errors = [
             score.error.error_repr if score.error else None
             for _, score in grouped_data_item
         ]
-        mean_inference = (
-            math.nan
-            if any(math.isnan(inference) for inference in inferences)
-            else float(np.mean(cast(List[float], inferences)))
-        )
-        # fill Nones with 0.0
-        min_possible_inference = float(
-            np.nan_to_num(np.array(inferences, dtype=float), nan=0.0).mean()
-        )
-        # fill Nones with 1.0
-        max_possible_inference = float(
-            np.nan_to_num(np.array(inferences, dtype=float), nan=1.0).mean()
-        )
         inference_returns.append(
             InferenceReturn(
                 inference=mean_inference,