aidial-rag-eval 0.6.0.dev19__tar.gz → 0.6.0.dev20__tar.gz

{aidial_rag_eval-0.6.0.dev19 → aidial_rag_eval-0.6.0.dev20}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: aidial-rag-eval
-Version: 0.6.0.dev19
+Version: 0.6.0.dev20
 Summary: Library for RAG evaluation
 License: Apache-2.0
 License-File: LICENSE
@@ -77,7 +77,8 @@ import pandas as pd
 from langchain_openai import AzureChatOpenAI
 from aidial_rag_eval import create_rag_eval_metrics_report
 from aidial_rag_eval.metric_binds import CONTEXT_TO_ANSWER_INFERENCE,\
-    ANSWER_TO_GROUND_TRUTH_INFERENCE, GROUND_TRUTH_TO_ANSWER_INFERENCE
+    ANSWER_TO_GROUND_TRUTH_INFERENCE, GROUND_TRUTH_TO_ANSWER_INFERENCE,\
+    ANSWER_TO_FACTS_INFERENCE, FACTS_TO_ANSWER_INFERENCE
 
 llm = AzureChatOpenAI(model="gemini-2.5-flash-lite")
 
@@ -108,16 +109,18 @@ df_metrics = create_rag_eval_metrics_report(
         CONTEXT_TO_ANSWER_INFERENCE,
         ANSWER_TO_GROUND_TRUTH_INFERENCE,
         GROUND_TRUTH_TO_ANSWER_INFERENCE,
+        ANSWER_TO_FACTS_INFERENCE,
+        FACTS_TO_ANSWER_INFERENCE,
     ],
 )
-print(df_metrics[["facts_ranks", "recall", 'precision', 'mrr', 'f1', 'ctx_ans_inference', 'ans_gt_inference', 'gt_ans_inference']])
+print(df_metrics[["facts_ranks", "recall", 'precision', 'mrr', 'f1', 'ctx_ans_inference', 'ans_gt_inference', 'gt_ans_inference', 'ans_fct_inference', 'fct_ans_inference']])
 ```
 
 It is expected to see the following results:
 
-| recall | precision | mrr | f1  | ctx_ans_inference | ans_gt_inference | gt_ans_inference |
-| ------ | --------- | --- | --- | ----------------- | ---------------- | ---------------- |
-| 0.5    | 0.5       | 0.5 | 0.5 | 1.0               | 0.5              | 1.0              |
+| recall | precision | mrr | f1  | ctx_ans_inference | ans_gt_inference | gt_ans_inference | ans_fct_inference | fct_ans_inference |
+| ------ | --------- | --- | --- | ----------------- | ---------------- | ---------------- | ----------------- | ----------------- |
+| 0.5    | 0.5       | 0.5 | 0.5 | 1.0               | 0.5              | 1.0              | 0.5               | 1.0               |
 
 In this table:
 
@@ -127,8 +130,12 @@ In this table:
   - "ctx" refers to 'context'
   - "ans" refers to 'answer'
   - "gt" refers to 'ground truth answer'
-- "ctx_ans_inference" and "ans_gt_inference" values of 1.0 mean our answer can be derived directly from the context and the ground truth answer, respectively.
-- "gt_ans_inference" of 0.5, denotes that the ground truth answer can only be partially inferred from our answer.
+  - "fct" refers to 'facts'
+- "ctx_ans_inference" of 1.0 means our answer can be fully derived from the context.
+- "ans_gt_inference" of 0.5 means the ground truth answer is only partially entailed by our answer.
+- "gt_ans_inference" of 1.0 means our answer can be fully derived from the ground truth answer.
+- "ans_fct_inference" of 0.5 means only half of the ground truth facts are entailed by our answer (the Pacific Ocean fact is missing).
+- "fct_ans_inference" of 1.0 means our answer can be fully derived from the ground truth facts.
 
 ## Recommended models
 

{aidial_rag_eval-0.6.0.dev19 → aidial_rag_eval-0.6.0.dev20}/README.md

@@ -44,7 +44,8 @@ import pandas as pd
 from langchain_openai import AzureChatOpenAI
 from aidial_rag_eval import create_rag_eval_metrics_report
 from aidial_rag_eval.metric_binds import CONTEXT_TO_ANSWER_INFERENCE,\
-    ANSWER_TO_GROUND_TRUTH_INFERENCE, GROUND_TRUTH_TO_ANSWER_INFERENCE
+    ANSWER_TO_GROUND_TRUTH_INFERENCE, GROUND_TRUTH_TO_ANSWER_INFERENCE,\
+    ANSWER_TO_FACTS_INFERENCE, FACTS_TO_ANSWER_INFERENCE
 
 llm = AzureChatOpenAI(model="gemini-2.5-flash-lite")
 
@@ -75,16 +76,18 @@ df_metrics = create_rag_eval_metrics_report(
         CONTEXT_TO_ANSWER_INFERENCE,
         ANSWER_TO_GROUND_TRUTH_INFERENCE,
         GROUND_TRUTH_TO_ANSWER_INFERENCE,
+        ANSWER_TO_FACTS_INFERENCE,
+        FACTS_TO_ANSWER_INFERENCE,
     ],
 )
-print(df_metrics[["facts_ranks", "recall", 'precision', 'mrr', 'f1', 'ctx_ans_inference', 'ans_gt_inference', 'gt_ans_inference']])
+print(df_metrics[["facts_ranks", "recall", 'precision', 'mrr', 'f1', 'ctx_ans_inference', 'ans_gt_inference', 'gt_ans_inference', 'ans_fct_inference', 'fct_ans_inference']])
 ```
 
 It is expected to see the following results:
 
-| recall | precision | mrr | f1  | ctx_ans_inference | ans_gt_inference | gt_ans_inference |
-| ------ | --------- | --- | --- | ----------------- | ---------------- | ---------------- |
-| 0.5    | 0.5       | 0.5 | 0.5 | 1.0               | 0.5              | 1.0              |
+| recall | precision | mrr | f1  | ctx_ans_inference | ans_gt_inference | gt_ans_inference | ans_fct_inference | fct_ans_inference |
+| ------ | --------- | --- | --- | ----------------- | ---------------- | ---------------- | ----------------- | ----------------- |
+| 0.5    | 0.5       | 0.5 | 0.5 | 1.0               | 0.5              | 1.0              | 0.5               | 1.0               |
 
 In this table:
 
@@ -94,8 +97,12 @@ In this table:
   - "ctx" refers to 'context'
   - "ans" refers to 'answer'
   - "gt" refers to 'ground truth answer'
-- "ctx_ans_inference" and "ans_gt_inference" values of 1.0 mean our answer can be derived directly from the context and the ground truth answer, respectively.
-- "gt_ans_inference" of 0.5, denotes that the ground truth answer can only be partially inferred from our answer.
+  - "fct" refers to 'facts'
+- "ctx_ans_inference" of 1.0 means our answer can be fully derived from the context.
+- "ans_gt_inference" of 0.5 means the ground truth answer is only partially entailed by our answer.
+- "gt_ans_inference" of 1.0 means our answer can be fully derived from the ground truth answer.
+- "ans_fct_inference" of 0.5 means only half of the ground truth facts are entailed by our answer (the Pacific Ocean fact is missing).
+- "fct_ans_inference" of 1.0 means our answer can be fully derived from the ground truth facts.
 
 ## Recommended models
 

{aidial_rag_eval-0.6.0.dev19 → aidial_rag_eval-0.6.0.dev20}/pyproject.toml

@@ -1,6 +1,6 @@
 [tool.poetry]
 name = "aidial-rag-eval"
-version = "0.6.0.dev19"
+version = "0.6.0.dev20"
 description = "Library for RAG evaluation"
 authors = ["EPAM RAIL <SpecialEPM-DIALDevTeam@epam.com>"]
 homepage = "https://github.com/epam/ai-dial-rag-eval"

{aidial_rag_eval-0.6.0.dev19 → aidial_rag_eval-0.6.0.dev20}/src/aidial_rag_eval/dataframe/metrics.py

@@ -8,13 +8,20 @@ from aidial_rag_eval.dataframe.match_facts import (
     apply_matcher_to_merged_dataframe,
 )
 from aidial_rag_eval.dataframe.merge import merge_ground_truth_and_answers
-from aidial_rag_eval.generation.metric_binds import metric_binds_dict
-from aidial_rag_eval.generation.types import MetricBind, inference_column
+from aidial_rag_eval.generation.metric_binds import (
+    _wrapped_dataframe_inference,
+    _wrapped_dataframe_refusal,
+)
+from aidial_rag_eval.generation.types import (
+    InferenceMetricBind,
+    MetricBind,
+    RefusalMetricBind,
+    inference_column,
+)
 from aidial_rag_eval.retrieval.metrics import (
     calculate_metrics as calculate_metrics_by_row,
 )
 from aidial_rag_eval.retrieval.types import Matcher
-from aidial_rag_eval.types import MergedColumns
 
 
 def apply_metrics_to_matched_results(match_result_data: pd.DataFrame) -> pd.DataFrame:
@@ -129,7 +136,7 @@ def calculate_generation_metrics(
         A Langchain chat model used for calculating generation metrics.
 
     metric_binds : List[MetricBind], optional, default=None
-        A list of string constants from `aidial_rag_eval.metric_binds`
+        A list of metric bind instances from `aidial_rag_eval.metric_binds`
         specifying the generation metrics.
 
     max_concurrency : int, default=8
@@ -144,19 +151,32 @@ def calculate_generation_metrics(
         Returns generation metrics dataframe.
     """
     df_merged_copy = df_merged.copy()
-    df_merged_copy[MergedColumns.JOINED_CONTEXT] = df_merged_copy[
-        MergedColumns.CONTEXT
-    ].apply(lambda x: "\n".join(x))
     metric_results = dict()
     for metric_bind in metric_binds:
-        metric_results.update(
-            metric_binds_dict[metric_bind](
+        if isinstance(metric_bind, InferenceMetricBind):
+            result = _wrapped_dataframe_inference(
+                df_merged=df_merged_copy,
+                premise_column=metric_bind.premise_column,
+                hypothesis_column=metric_bind.hypothesis_column,
+                llm=llm,
+                prefix=metric_bind.prefix,
+                premise_prefix_column=metric_bind.premise_prefix_column,
+                document_column=metric_bind.document_column,
+                max_concurrency=max_concurrency,
+                show_progress_bar=show_progress_bar,
+            )
+        elif isinstance(metric_bind, RefusalMetricBind):
+            result = _wrapped_dataframe_refusal(
                 df_merged=df_merged_copy,
+                answer_column=metric_bind.answer_column,
                 llm=llm,
+                prefix=metric_bind.prefix,
                 max_concurrency=max_concurrency,
                 show_progress_bar=show_progress_bar,
-            ).to_dict(orient="series")
-        )
+            )
+        else:
+            raise ValueError(f"Unknown metric bind type: {type(metric_bind)}")
+        metric_results.update(result.to_dict(orient="series"))
     df_metrics = pd.DataFrame(data=metric_results)
     nli_columns = [
         column for column in metric_results.keys() if column.endswith(inference_column)
@@ -198,7 +218,7 @@ def create_generation_metrics_report(
         A Langchain chat model used for calculating generation metrics.
 
     metric_binds : List[MetricBind], optional, default=None
-        A list of string constants from `aidial_rag_eval.metric_binds`
+        A list of metric bind instances from `aidial_rag_eval.metric_binds`
         specifying the generation metrics.
 
     max_concurrency : int, default=8
@@ -256,7 +276,7 @@ def create_rag_eval_metrics_report(
         A Langchain chat model used for calculating generation metrics.
 
     metric_binds : List[MetricBind], optional, default=None
-        A list of string constants from `aidial_rag_eval.metric_binds`
+        A list of metric bind instances from `aidial_rag_eval.metric_binds`
         specifying the generation metrics.
 
     max_concurrency : int, default=8

{aidial_rag_eval-0.6.0.dev19 → aidial_rag_eval-0.6.0.dev20}/src/aidial_rag_eval/generation/inference.py

@@ -30,13 +30,15 @@ from aidial_rag_eval.types import Documents, Question
 
 T = TypeVar("T")
 
+_LIST_DELIMITER = "\n"
+
 
 def _join_documents(documents: Documents) -> JoinedDocumentsName:
     return " ; ".join(documents)
 
 
 def _make_inference_task_inputs(
-    premises: List[Premise],
+    adjusted_premises: List[str],
     statements: List[Union[HypothesisStatements, ErrorInfo]],
     document_names: List[JoinedDocumentsName],
 ) -> List[InferenceInputs]:
@@ -45,7 +47,7 @@ def _make_inference_task_inputs(
 
     Parameters
     -----------
-    premises : List[str]
+    adjusted_premises : List[str]
         A list of premises from which we want to derive hypotheses in pairs.
 
     statements : List[Union[HypothesisStatements, ErrorInfo]]
@@ -73,7 +75,7 @@ def _make_inference_task_inputs(
                 [
                     InferenceInputs(
                         hypothesis_id=i,
-                        premise=premises[i],
+                        premise=adjusted_premises[i],
                         statements=list_statements,
                         document_name=document_names[i],
                     )
@@ -83,7 +85,7 @@ def _make_inference_task_inputs(
                 else [
                     InferenceInputs(
                         hypothesis_id=i,
-                        premise=premises[i],
+                        premise=adjusted_premises[i],
                         statements=[],
                         document_name=document_names[i],
                         error=statement_result,
@@ -211,8 +213,9 @@ def _segment_hypotheses(
     Parameters
     -----------
 
-    hypotheses : List[str]
+    hypotheses : List[Union[str, List[str]]]
         The text of the hypothesis.
+        Each hypothesis can be a single text or a list of text fragments.
 
     llm : BaseChatModel
         The Langchain chat model used for calculating inference.
@@ -235,7 +238,12 @@ def _segment_hypotheses(
         max_concurrency=max_concurrency,
     )
     segmented_hypotheses = [
-        SegmentedText.from_text(text=hypothesis) for hypothesis in hypotheses
+        (
+            SegmentedText.from_list(hypothesis, _LIST_DELIMITER)
+            if not isinstance(hypothesis, str)
+            else SegmentedText.from_text(hypothesis)
+        )
+        for hypothesis in hypotheses
     ]
     if show_progress_bar:
         print("Converting hypothesis...")
@@ -289,15 +297,17 @@ def _extract_statements(
     return extractor.extract(segmented_hypotheses, show_progress_bar)
 
 
-def _add_questions_to_premises(
-    premises: List[Premise],
-    questions: Optional[List[Question]],
-) -> List[Premise]:
-    adjusted_premises: List[Premise] = list(premises)
-    if questions is not None:
-        for i, question in enumerate(questions):
-            question_split = SegmentedText.from_text(text=question)
-            adjusted_premises[i] = question_split.segments[-1] + "\n" + premises[i]
+def _prepend_premise_prefixes(
+    normalized_premises: List[str],
+    premise_prefixes: Optional[List[Question]],
+) -> List[str]:
+    adjusted_premises: List[str] = list(normalized_premises)
+    if premise_prefixes is not None:
+        for i, prefix in enumerate(premise_prefixes):
+            prefix_split = SegmentedText.from_text(text=prefix)
+            adjusted_premises[i] = _LIST_DELIMITER.join(
+                [prefix_split.segments[-1], normalized_premises[i]]
+            )
     return adjusted_premises
 
 
@@ -305,7 +315,7 @@ def _infer_statements(
     premises: List[Premise],
     statements: List[Union[HypothesisStatements, ErrorInfo]],
     llm: BaseChatModel,
-    questions: Optional[List[Question]] = None,
+    premise_prefixes: Optional[List[Question]] = None,
     list_documents: Optional[List[Documents]] = None,
     max_concurrency: int = 8,
     show_progress_bar: bool = True,
@@ -319,8 +329,9 @@ def _infer_statements(
     Parameters
     -----------
 
-    premises : List[str]
+    premises : List[Union[str, List[str]]]
         The text of the premise from which the hypothesis will be inferred.
+        Each premise can be a single text or a list of text fragments.
 
     statements : List[Union[HypothesisStatements, ErrorInfo]]
         A deeply nested list of statements, where the outermost
@@ -352,7 +363,10 @@ def _infer_statements(
         For items that failed in any stage, InferenceScore will have inference=None
         and the error field set.
     """
-    adjusted_premises = _add_questions_to_premises(premises, questions)
+    normalized_premises: List[str] = [
+        _LIST_DELIMITER.join(p) if not isinstance(p, str) else p for p in premises
+    ]
+    adjusted_premises = _prepend_premise_prefixes(normalized_premises, premise_prefixes)
 
     document_names: List[JoinedDocumentsName] = (
         [""] * len(statements)
@@ -412,7 +426,7 @@ def infer_statements(
     premises: List[Premise],
     statements: List[HypothesisStatements],
     llm: BaseChatModel,
-    questions: Optional[List[Question]] = None,
+    premise_prefixes: Optional[List[Question]] = None,
     list_documents: Optional[List[Documents]] = None,
     max_concurrency: int = 8,
     show_progress_bar: bool = True,
@@ -421,7 +435,7 @@ def infer_statements(
         premises=premises,
         statements=cast(List[Union[HypothesisStatements, ErrorInfo]], statements),
         llm=llm,
-        questions=questions,
+        premise_prefixes=premise_prefixes,
         list_documents=list_documents,
         max_concurrency=max_concurrency,
         show_progress_bar=show_progress_bar,
@@ -468,7 +482,7 @@ def calculate_batch_inference(
     premises: List[Premise],
     hypotheses: List[Hypothesis],
     llm: BaseChatModel,
-    questions: Optional[List[Question]] = None,
+    premise_prefixes: Optional[List[Question]] = None,
     list_documents: Optional[List[Documents]] = None,
     max_concurrency: int = 8,
     show_progress_bar: bool = True,
@@ -479,11 +493,13 @@ def calculate_batch_inference(
     Parameters
     -----------
 
-    premises : List[str]
+    premises : List[Union[str, List[str]]]
         The text of the premise from which the hypothesis will be inferred.
+        Each premise can be a single text or a list of text fragments.
 
-    hypotheses : List[str]
+    hypotheses : List[Union[str, List[str]]]
         The text of the hypothesis.
+        Each hypothesis can be a single text or a list of text fragments.
 
     llm : BaseChatModel
         The Langchain chat model used for calculating inference.
@@ -529,7 +545,7 @@ def calculate_batch_inference(
             premises=premises,
             statements=statements,
             llm=llm,
-            questions=questions,
+            premise_prefixes=premise_prefixes,
             list_documents=list_documents,
             max_concurrency=max_concurrency,
             show_progress_bar=show_progress_bar,
@@ -566,7 +582,7 @@ def calculate_inference(
     premise: Premise,
     hypothesis: Hypothesis,
     llm: BaseChatModel,
-    question: Optional[Question] = None,
+    premise_prefix: Optional[Question] = None,
     documents: Optional[Documents] = None,
     max_concurrency: int = 8,
     show_progress_bar: bool = True,
@@ -577,11 +593,13 @@ def calculate_inference(
     Parameters
     -----------
 
-    premise : str
+    premise : Union[str, List[str]]
         The text of the premise from which the hypothesis will be inferred.
+        Can be a single text or a list of text fragments.
 
-    hypothesis : str
+    hypothesis : Union[str, List[str]]
         The text of the hypothesis.
+        Can be a single text or a list of text fragments.
 
     llm : BaseChatModel
         The Langchain chat model used for calculating inference.
@@ -605,13 +623,13 @@ def calculate_inference(
         along with a JSON string that explains how the inference was derived and
         highlights string used for highlighting each segment of the hypothesis.
     """
-    questions = None if question is None else [question]
+    premise_prefixes = None if premise_prefix is None else [premise_prefix]
     list_documents = None if documents is None else [documents]
     inference_returns = calculate_batch_inference(
         premises=[premise],
         hypotheses=[hypothesis],
         llm=llm,
-        questions=questions,
+        premise_prefixes=premise_prefixes,
         list_documents=list_documents,
         max_concurrency=max_concurrency,
         show_progress_bar=show_progress_bar,

aidial_rag_eval-0.6.0.dev20/src/aidial_rag_eval/generation/metric_binds.py

@@ -0,0 +1,125 @@
+import dataclasses
+from typing import Any, List, Optional
+
+import numpy as np
+import pandas as pd
+from langchain_core.language_models import BaseChatModel
+
+from aidial_rag_eval.generation.inference import calculate_batch_inference
+from aidial_rag_eval.generation.refusal import calculate_batch_refusal
+from aidial_rag_eval.generation.types import InferenceMetricBind, RefusalMetricBind
+from aidial_rag_eval.types import MergedColumns
+
+C2A_INFERENCE_PREFIX = "ctx_ans_"
+A2GT_INFERENCE_PREFIX = "ans_gt_"
+GT2A_INFERENCE_PREFIX = "gt_ans_"
+A2F_INFERENCE_PREFIX = "ans_fct_"
+F2A_INFERENCE_PREFIX = "fct_ans_"
+
+ANSWER_REFUSAL_PREFIX = "answer_"
+GT_ANSWER_REFUSAL_PREFIX = "ground_truth_"
+
+
+def _get_column_as_list(dataframe: pd.DataFrame, column: str) -> List[Any]:
+    return [x.tolist() if isinstance(x, np.ndarray) else x for x in dataframe[column]]
+
+
+def _wrapped_dataframe_inference(
+    df_merged: pd.DataFrame,
+    premise_column: str,
+    hypothesis_column: str,
+    llm: BaseChatModel,
+    prefix: str,
+    premise_prefix_column: Optional[str] = None,
+    document_column: Optional[str] = None,
+    max_concurrency: int = 8,
+    show_progress_bar: bool = True,
+) -> pd.DataFrame:
+    inference_returns = calculate_batch_inference(
+        premises=_get_column_as_list(df_merged, premise_column),
+        hypotheses=_get_column_as_list(df_merged, hypothesis_column),
+        llm=llm,
+        premise_prefixes=(
+            _get_column_as_list(df_merged, premise_prefix_column)
+            if premise_prefix_column is not None
+            else None
+        ),
+        list_documents=(
+            _get_column_as_list(df_merged, document_column)
+            if document_column is not None
+            else None
+        ),
+        max_concurrency=max_concurrency,
+        show_progress_bar=show_progress_bar,
+    )
+    return pd.DataFrame(
+        [dataclasses.asdict(inference_return) for inference_return in inference_returns]
+    ).add_prefix(prefix)
+
+
+def _wrapped_dataframe_refusal(
+    df_merged: pd.DataFrame,
+    answer_column: str,
+    llm: BaseChatModel,
+    prefix: str,
+    max_concurrency: int = 8,
+    show_progress_bar: bool = True,
+) -> pd.DataFrame:
+    refusal_returns = calculate_batch_refusal(
+        answers=_get_column_as_list(df_merged, answer_column),
+        llm=llm,
+        max_concurrency=max_concurrency,
+        show_progress_bar=show_progress_bar,
+    )
+    return pd.DataFrame(
+        [dataclasses.asdict(refusal) for refusal in refusal_returns]
+    ).add_prefix(prefix)
+
+
+CONTEXT_TO_ANSWER_INFERENCE = InferenceMetricBind(
+    premise_column=MergedColumns.CONTEXT,
+    hypothesis_column=MergedColumns.ANSWER,
+    prefix=C2A_INFERENCE_PREFIX,
+    document_column=MergedColumns.DOCUMENTS,
+)
+
+ANSWER_TO_GROUND_TRUTH_INFERENCE = InferenceMetricBind(
+    premise_column=MergedColumns.ANSWER,
+    hypothesis_column=MergedColumns.GROUND_TRUTH_ANSWER,
+    prefix=A2GT_INFERENCE_PREFIX,
+    premise_prefix_column=MergedColumns.QUESTION,
+    document_column=MergedColumns.DOCUMENTS,
+)
+
+GROUND_TRUTH_TO_ANSWER_INFERENCE = InferenceMetricBind(
+    premise_column=MergedColumns.GROUND_TRUTH_ANSWER,
+    hypothesis_column=MergedColumns.ANSWER,
+    prefix=GT2A_INFERENCE_PREFIX,
+    premise_prefix_column=MergedColumns.QUESTION,
+    document_column=MergedColumns.DOCUMENTS,
+)
+
+ANSWER_TO_FACTS_INFERENCE = InferenceMetricBind(
+    premise_column=MergedColumns.ANSWER,
+    hypothesis_column=MergedColumns.FACTS,
+    prefix=A2F_INFERENCE_PREFIX,
+    premise_prefix_column=MergedColumns.QUESTION,
+    document_column=MergedColumns.DOCUMENTS,
+)
+
+FACTS_TO_ANSWER_INFERENCE = InferenceMetricBind(
+    premise_column=MergedColumns.FACTS,
+    hypothesis_column=MergedColumns.ANSWER,
+    prefix=F2A_INFERENCE_PREFIX,
+    document_column=MergedColumns.DOCUMENTS,
+)
+
+ANSWER_REFUSAL = RefusalMetricBind(
+    answer_column=MergedColumns.ANSWER,
+    prefix=ANSWER_REFUSAL_PREFIX,
+)
+
+GROUND_TRUTH_REFUSAL = RefusalMetricBind(
+    answer_column=MergedColumns.GROUND_TRUTH_ANSWER,
+    prefix=GT_ANSWER_REFUSAL_PREFIX,
+)

{aidial_rag_eval-0.6.0.dev19 → aidial_rag_eval-0.6.0.dev20}/src/aidial_rag_eval/generation/types.py

@@ -3,19 +3,32 @@ import json
 from dataclasses import dataclass, fields
 from typing import List, Optional, Union
 
-from aidial_rag_eval.types import Answer, GroundTruthAnswer, Text
-
 TextSegment = str
-JoinedContext = Text
 
-Premise = Union[JoinedContext, Answer, GroundTruthAnswer]
-Hypothesis = Union[Answer, GroundTruthAnswer]
+Premise = Union[str, List[str]]
+Hypothesis = Union[str, List[str]]
 Statement = str
 HypothesisSegmentStatements = List[Statement]
 HypothesisStatements = List[HypothesisSegmentStatements]
 JoinedDocumentsName = str
 
-MetricBind = str
+
+@dataclass(frozen=True)
+class InferenceMetricBind:
+    premise_column: str
+    hypothesis_column: str
+    prefix: str
+    premise_prefix_column: Optional[str] = None
+    document_column: Optional[str] = None
+
+
+@dataclass(frozen=True)
+class RefusalMetricBind:
+    answer_column: str
+    prefix: str
+
+
+MetricBind = Union[InferenceMetricBind, RefusalMetricBind]
 
 
 @dataclass

{aidial_rag_eval-0.6.0.dev19 → aidial_rag_eval-0.6.0.dev20}/src/aidial_rag_eval/generation/utils/segmented_text.py

@@ -3,7 +3,7 @@ from typing import Callable, List, Optional, Tuple
 import spacy
 from more_itertools import chunked, constrained_batches
 
-from aidial_rag_eval.generation.types import Text, TextSegment
+from aidial_rag_eval.generation.types import TextSegment
 
 Splitter = Callable[[TextSegment], List[TextSegment]]
 
@@ -30,7 +30,7 @@ def _sent_tokenize(text: str) -> List[str]:
 SegmentChecker = Callable[[TextSegment], bool]
 
 
-def get_delimiters(text: Text, segmented_text: List[TextSegment]) -> List[str]:
+def get_delimiters(text: str, segmented_text: List[TextSegment]) -> List[str]:
     delimiters = []
     start = 0
     for i, part in enumerate(segmented_text[:-1]):
@@ -48,7 +48,7 @@ def get_delimiters(text: Text, segmented_text: List[TextSegment]) -> List[str]:
 
 def join_with_delimiters(
     segmented_text: List[TextSegment], delimiters: List[str]
-) -> Text:
+) -> str:
     result = []
     for i in range(len(segmented_text)):
         result.append(segmented_text[i])
@@ -101,7 +101,18 @@ class SegmentedText:
         self.delimiters = delimiters.copy()
 
     @classmethod
-    def from_text(cls, text: Text) -> "SegmentedText":
+    def from_list(cls, texts: List[str], delimiter: str = "\n") -> "SegmentedText":
+        segmented = [cls.from_text(text) for text in texts]
+        segments = segmented[0].segments.copy()
+        delimiters = segmented[0].delimiters.copy()
+        for st in segmented[1:]:
+            delimiters.append(delimiter)
+            segments.extend(st.segments)
+            delimiters.extend(st.delimiters)
+        return cls(segments, delimiters)
+
+    @classmethod
+    def from_text(cls, text: str) -> "SegmentedText":
         max_len = 500
         min_len = 10
         conditional_splitters: List[Splitter] = [

aidial_rag_eval-0.6.0.dev20/src/aidial_rag_eval/metric_binds.py

@@ -0,0 +1,19 @@
+from aidial_rag_eval.generation.metric_binds import (
+    ANSWER_REFUSAL,
+    ANSWER_TO_FACTS_INFERENCE,
+    ANSWER_TO_GROUND_TRUTH_INFERENCE,
+    CONTEXT_TO_ANSWER_INFERENCE,
+    FACTS_TO_ANSWER_INFERENCE,
+    GROUND_TRUTH_REFUSAL,
+    GROUND_TRUTH_TO_ANSWER_INFERENCE,
+)
+
+__all__ = [
+    "CONTEXT_TO_ANSWER_INFERENCE",
+    "ANSWER_TO_GROUND_TRUTH_INFERENCE",
+    "ANSWER_TO_FACTS_INFERENCE",
+    "FACTS_TO_ANSWER_INFERENCE",
+    "GROUND_TRUTH_TO_ANSWER_INFERENCE",
+    "ANSWER_REFUSAL",
+    "GROUND_TRUTH_REFUSAL",
+]

{aidial_rag_eval-0.6.0.dev19 → aidial_rag_eval-0.6.0.dev20}/src/aidial_rag_eval/types.py

@@ -5,14 +5,13 @@ from typing import List, TypeVar
 FactType = TypeVar("FactType")
 ContextChunk = str
 Document = str
-Text = str
 
 Context = List[ContextChunk]
 Facts = List[FactType]
 Documents = List[Document]
-Question = Text
-Answer = Text
-GroundTruthAnswer = Text
+Question = str
+Answer = str
+GroundTruthAnswer = str
 
 
 @dataclass
@@ -52,7 +51,6 @@ class MergedColumns(str, Enum):
     GROUND_TRUTH_ANSWER = "ground_truth_answer"
     DOCUMENTS = GroundTruthColumns.DOCUMENTS.value
     FACTS = GroundTruthColumns.FACTS.value
-    JOINED_CONTEXT = "joined_context"
 
 
 MERGED_KEY_COLUMNS = [MergedColumns.DOCUMENTS, MergedColumns.QUESTION]

aidial_rag_eval-0.6.0.dev19/src/aidial_rag_eval/generation/metric_binds.py

@@ -1,165 +0,0 @@
-import dataclasses
-from typing import Any, Callable, Dict, List, Optional
-
-import pandas as pd
-from langchain_core.language_models import BaseChatModel
-
-from aidial_rag_eval.generation.inference import calculate_batch_inference
-from aidial_rag_eval.generation.refusal import calculate_batch_refusal
-from aidial_rag_eval.generation.types import MetricBind
-from aidial_rag_eval.types import MergedColumns
-
-C2A_INFERENCE_PREFIX = "ctx_ans_"
-A2GT_INFERENCE_PREFIX = "ans_gt_"
-GT2A_INFERENCE_PREFIX = "gt_ans_"
-
-ANSWER_REFUSAL_PREFIX = "answer_"
-GT_ANSWER_REFUSAL_PREFIX = "ground_truth_"
-
-
-def _get_column_as_list_str(dataframe: pd.DataFrame, column: str) -> List[Any]:
-    list_str = dataframe[column].to_list()
-    assert isinstance(list_str, list)
-    return list_str
-
-
-def _wrapped_dataframe_inference(
-    df_merged: pd.DataFrame,
-    premise_column: str,
-    hypothesis_column: str,
-    llm: BaseChatModel,
-    prefix: str,
-    question_column: Optional[str] = None,
-    document_column: Optional[str] = None,
-    max_concurrency: int = 8,
-    show_progress_bar: bool = True,
-) -> pd.DataFrame:
-    inference_returns = calculate_batch_inference(
-        premises=_get_column_as_list_str(df_merged, premise_column),
-        hypotheses=_get_column_as_list_str(df_merged, hypothesis_column),
-        llm=llm,
-        questions=(
-            _get_column_as_list_str(df_merged, question_column)
-            if question_column is not None
-            else None
-        ),
-        list_documents=(
-            _get_column_as_list_str(df_merged, document_column)
-            if document_column is not None
-            else None
-        ),
-        max_concurrency=max_concurrency,
-        show_progress_bar=show_progress_bar,
-    )
-    return pd.DataFrame(
-        [dataclasses.asdict(inference_return) for inference_return in inference_returns]
-    ).add_prefix(prefix)
-
-
-def _wrapped_dataframe_refusal(
-    df_merged: pd.DataFrame,
-    answer_column: str,
-    llm: BaseChatModel,
-    prefix: str,
-    max_concurrency: int = 8,
-    show_progress_bar: bool = True,
-) -> pd.DataFrame:
-    refusal_returns = calculate_batch_refusal(
-        answers=_get_column_as_list_str(df_merged, answer_column),
-        llm=llm,
-        max_concurrency=max_concurrency,
-        show_progress_bar=show_progress_bar,
-    )
-    return pd.DataFrame(
-        [dataclasses.asdict(refusal) for refusal in refusal_returns]
-    ).add_prefix(prefix)
-
-
-def context_to_answer_inference(
-    df_merged, llm, max_concurrency, show_progress_bar, **kwargs
-) -> pd.DataFrame:
-    return _wrapped_dataframe_inference(
-        df_merged=df_merged,
-        premise_column=MergedColumns.JOINED_CONTEXT,
-        hypothesis_column=MergedColumns.ANSWER,
-        llm=llm,
-        prefix=C2A_INFERENCE_PREFIX,
-        # The last segment(sentence) of the question is attached to the premise.
-        # This can be helpful when the premise is the answer or ground truth
-        # and it is simple. Example:
-        # question: how many boxes are in the cupboard?
-        # answer (premise): 3.
-        # When the premise is the context, the question is not needed.
-        question_column=None,
-        document_column=MergedColumns.DOCUMENTS,
-        max_concurrency=max_concurrency,
-        show_progress_bar=show_progress_bar,
-    )
-
-
-def answer_to_ground_truth_inference(
-    df_merged, llm, max_concurrency, show_progress_bar, **kwargs
-) -> pd.DataFrame:
-    return _wrapped_dataframe_inference(
-        df_merged=df_merged,
-        premise_column=MergedColumns.ANSWER,
-        hypothesis_column=MergedColumns.GROUND_TRUTH_ANSWER,
-        llm=llm,
-        prefix=A2GT_INFERENCE_PREFIX,
-        question_column=MergedColumns.QUESTION,
-        document_column=MergedColumns.DOCUMENTS,
-        max_concurrency=max_concurrency,
-        show_progress_bar=show_progress_bar,
-    )
-
-
-def ground_truth_to_answer_inference(
-    df_merged, llm, max_concurrency, show_progress_bar, **kwargs
-) -> pd.DataFrame:
-    return _wrapped_dataframe_inference(
-        df_merged=df_merged,
-        premise_column=MergedColumns.GROUND_TRUTH_ANSWER,
-        hypothesis_column=MergedColumns.ANSWER,
-        llm=llm,
-        prefix=GT2A_INFERENCE_PREFIX,
-        question_column=MergedColumns.QUESTION,
-        document_column=MergedColumns.DOCUMENTS,
-        max_concurrency=max_concurrency,
-        show_progress_bar=show_progress_bar,
-    )
-
-
-def answer_refusal(
-    df_merged, llm, max_concurrency, show_progress_bar, **kwargs
-) -> pd.DataFrame:
-    return _wrapped_dataframe_refusal(
-        df_merged=df_merged,
-        answer_column=MergedColumns.ANSWER,
-        llm=llm,
-        prefix=ANSWER_REFUSAL_PREFIX,
-        max_concurrency=max_concurrency,
-        show_progress_bar=show_progress_bar,
-    )
-
-
-def ground_truth_refusal(
-    df_merged, llm, max_concurrency, show_progress_bar, **kwargs
-) -> pd.DataFrame:
-    return _wrapped_dataframe_refusal(
-        df_merged=df_merged,
-        answer_column=MergedColumns.GROUND_TRUTH_ANSWER,
-        llm=llm,
-        prefix=GT_ANSWER_REFUSAL_PREFIX,
-        max_concurrency=max_concurrency,
-        show_progress_bar=show_progress_bar,
-    )
-
-
-metric_binds_dict: Dict[MetricBind, Callable] = {
-    "context_to_answer_inference": context_to_answer_inference,
-    "answer_to_ground_truth_inference": answer_to_ground_truth_inference,
-    "ground_truth_to_answer_inference": ground_truth_to_answer_inference,
-    "answer_refusal": answer_refusal,
-    "ground_truth_refusal": ground_truth_refusal,
-}
-metric_bind_keys: List[MetricBind] = list(metric_binds_dict.keys())

aidial_rag_eval-0.6.0.dev19/src/aidial_rag_eval/metric_binds.py

@@ -1,15 +0,0 @@
-from aidial_rag_eval.generation.metric_binds import metric_bind_keys
-
-CONTEXT_TO_ANSWER_INFERENCE = metric_bind_keys[0]
-ANSWER_TO_GROUND_TRUTH_INFERENCE = metric_bind_keys[1]
-GROUND_TRUTH_TO_ANSWER_INFERENCE = metric_bind_keys[2]
-ANSWER_REFUSAL = metric_bind_keys[3]
-GROUND_TRUTH_REFUSAL = metric_bind_keys[4]
-
-__all__ = [
-    "CONTEXT_TO_ANSWER_INFERENCE",
-    "ANSWER_TO_GROUND_TRUTH_INFERENCE",
-    "GROUND_TRUTH_TO_ANSWER_INFERENCE",
-    "ANSWER_REFUSAL",
-    "GROUND_TRUTH_REFUSAL",
-]