aidial-rag-eval 0.6.0.dev16__tar.gz → 0.6.0.dev19__tar.gz

{aidial_rag_eval-0.6.0.dev16 → aidial_rag_eval-0.6.0.dev19}/PKG-INFO

@@ -1,13 +1,13 @@
 Metadata-Version: 2.4
 Name: aidial-rag-eval
-Version: 0.6.0.dev16
+Version: 0.6.0.dev19
 Summary: Library for RAG evaluation
 License: Apache-2.0
 License-File: LICENSE
 Keywords: ai
 Author: EPAM RAIL
 Author-email: SpecialEPM-DIALDevTeam@epam.com
-Requires-Python: >=3.11,<4.0
+Requires-Python: >=3.11,<3.15
 Classifier: License :: OSI Approved :: Apache Software License
 Classifier: Programming Language :: Python :: 3
 Classifier: Programming Language :: Python :: 3.11
@@ -19,12 +19,12 @@ Provides-Extra: s3fs
 Requires-Dist: fsspec (>=2024.3.1,<2025.0.0)
 Requires-Dist: langchain-core (>=0.3.80,<2)
 Requires-Dist: more-itertools (>=10.5.0,<11.0.0)
-Requires-Dist: nltk (>=3.8.1,<4.0.0)
 Requires-Dist: numpy (>=1.26.4,<3)
 Requires-Dist: pandas (>=2.2.2,<3.0.0)
 Requires-Dist: pyarrow (>=20.0.0)
-Requires-Dist: pydantic (>=2.7.1,<3.0.0)
+Requires-Dist: pydantic (>=2.7.1)
 Requires-Dist: s3fs (>=2024.3.1,<2025.0.0) ; extra == "s3fs"
+Requires-Dist: spacy (>=3.8.9,<4)
 Requires-Dist: tqdm (>=4.66.4,<5.0.0)
 Project-URL: Documentation, https://github.com/epam/ai-dial-rag-eval
 Project-URL: Homepage, https://github.com/epam/ai-dial-rag-eval
@@ -47,6 +47,27 @@ Install the library using [pip](https://pip.pypa.org/en/stable/getting-started):
 pip install aidial-rag-eval
 ```
 
+### spaCy language model
+
+The generation metrics require the English language model for [spaCy](https://spacy.io/).
+Download it after installation:
+
+```sh
+python -m spacy download en_core_web_sm
+```
+
+Alternatively, you can install the model directly via URL:
+
+```sh
+pip install https://github.com/explosion/spacy-models/releases/download/en_core_web_sm-3.8.0/en_core_web_sm-3.8.0-py3-none-any.whl
+```
+
+Or as a [Poetry](https://python-poetry.org/) dependency:
+
+```toml
+en-core-web-sm = {url = "https://github.com/explosion/spacy-models/releases/download/en_core_web_sm-3.8.0/en_core_web_sm-3.8.0-py3-none-any.whl"}
+```
+
 ### Example
 
 The example of how to get retrieval metrics along with answer inference based on the context.
@@ -113,10 +134,11 @@ In this table:
 
 The algorithm is token-intensive. Considering the balance between quality and price, the following models are recommended:
 
+- **gemini-3.1-flash-lite**
 - **gemini-2.5-flash-lite**
 - **gpt-5-mini**
-- **gemini-2.0-flash-lite**
 - **gpt-5-nano**
+- **gpt-5.4-mini**
 
 ## Developer environment
 

{aidial_rag_eval-0.6.0.dev16 → aidial_rag_eval-0.6.0.dev19}/README.md

@@ -14,6 +14,27 @@ Install the library using [pip](https://pip.pypa.org/en/stable/getting-started):
 pip install aidial-rag-eval
 ```
 
+### spaCy language model
+
+The generation metrics require the English language model for [spaCy](https://spacy.io/).
+Download it after installation:
+
+```sh
+python -m spacy download en_core_web_sm
+```
+
+Alternatively, you can install the model directly via URL:
+
+```sh
+pip install https://github.com/explosion/spacy-models/releases/download/en_core_web_sm-3.8.0/en_core_web_sm-3.8.0-py3-none-any.whl
+```
+
+Or as a [Poetry](https://python-poetry.org/) dependency:
+
+```toml
+en-core-web-sm = {url = "https://github.com/explosion/spacy-models/releases/download/en_core_web_sm-3.8.0/en_core_web_sm-3.8.0-py3-none-any.whl"}
+```
+
 ### Example
 
 The example of how to get retrieval metrics along with answer inference based on the context.
@@ -80,10 +101,11 @@ In this table:
 
 The algorithm is token-intensive. Considering the balance between quality and price, the following models are recommended:
 
+- **gemini-3.1-flash-lite**
 - **gemini-2.5-flash-lite**
 - **gpt-5-mini**
-- **gemini-2.0-flash-lite**
 - **gpt-5-nano**
+- **gpt-5.4-mini**
 
 ## Developer environment
 

{aidial_rag_eval-0.6.0.dev16 → aidial_rag_eval-0.6.0.dev19}/pyproject.toml

@@ -1,6 +1,6 @@
 [tool.poetry]
 name = "aidial-rag-eval"
-version = "0.6.0.dev16"
+version = "0.6.0.dev19"
 description = "Library for RAG evaluation"
 authors = ["EPAM RAIL <SpecialEPM-DIALDevTeam@epam.com>"]
 homepage = "https://github.com/epam/ai-dial-rag-eval"
@@ -13,13 +13,13 @@ repository = "https://github.com/epam/ai-dial-rag-eval"
 
 
 [tool.poetry.dependencies]
-python = "^3.11"
+python = ">=3.11, <3.15"
 numpy = ">=1.26.4,<3"
 pandas = "^2.2.2"
 fsspec = "^2024.3.1"
 s3fs = {version = "^2024.3.1", optional = true}
-pydantic = "^2.7.1"
-nltk = "^3.8.1"
+pydantic = ">=2.7.1"
+spacy = ">=3.8.9,<4"
 tqdm = "^4.66.4"
 langchain-core = ">=0.3.80,<2"
 more-itertools = "^10.5.0"
@@ -35,6 +35,7 @@ pytest-cov = "^4.1.0"
 pytest-mock = "^3.14.0"
 langchain-community = "^0.3.14"
 langchain-openai = "^0.2.1"
+en-core-web-sm = {url = "https://github.com/explosion/spacy-models/releases/download/en_core_web_sm-3.8.0/en_core_web_sm-3.8.0-py3-none-any.whl"}
 
 [tool.poetry.group.lint.dependencies]
 black = "^24.3.0"

{aidial_rag_eval-0.6.0.dev16 → aidial_rag_eval-0.6.0.dev19}/src/aidial_rag_eval/dataframe/metrics.py

@@ -114,7 +114,6 @@ def calculate_generation_metrics(
     metric_binds: List[MetricBind],
     max_concurrency: int = 8,
     show_progress_bar: bool = True,
-    auto_download_nltk: bool = True,
 ) -> pd.DataFrame:
     """
     Calculates RAG evaluation generation metrics from df_merged dataframe.
@@ -156,7 +155,6 @@ def calculate_generation_metrics(
                 llm=llm,
                 max_concurrency=max_concurrency,
                 show_progress_bar=show_progress_bar,
-                auto_download_nltk=auto_download_nltk,
             ).to_dict(orient="series")
         )
     df_metrics = pd.DataFrame(data=metric_results)
@@ -180,7 +178,6 @@ def create_generation_metrics_report(
     metric_binds: List[MetricBind],
     max_concurrency: int = 8,
     show_progress_bar: bool = True,
-    auto_download_nltk: bool = True,
 ) -> pd.DataFrame:
     """
     Calculates RAG evaluation generation metrics from input dataframes.
@@ -223,7 +220,6 @@ def create_generation_metrics_report(
         metric_binds,
         max_concurrency,
         show_progress_bar,
-        auto_download_nltk,
     )
     return pd.merge(df_merged, df_metrics, left_index=True, right_index=True)
 
@@ -236,7 +232,6 @@ def create_rag_eval_metrics_report(
     metric_binds: Optional[List[MetricBind]] = None,
     max_concurrency: int = 8,
     show_progress_bar: bool = True,
-    auto_download_nltk: bool = True,
 ) -> pd.DataFrame:
     """
     Calculates RAG evaluation metrics from input dataframes.
@@ -287,6 +282,5 @@ def create_rag_eval_metrics_report(
         metric_binds,
         max_concurrency,
         show_progress_bar,
-        auto_download_nltk,
     )
     return pd.concat([df_merged, retrieval_metrics, generation_metrics], axis=1)

{aidial_rag_eval-0.6.0.dev16 → aidial_rag_eval-0.6.0.dev19}/src/aidial_rag_eval/evaluate.py

@@ -21,7 +21,6 @@ def evaluate(
     metric_binds: Optional[List[MetricBind]] = None,
     max_concurrency: int = 8,
     show_progress_bar: bool = True,
-    auto_download_nltk: bool = False,
 ) -> Dataset:
     """
     Calculates RAG evaluation metrics from input
@@ -83,7 +82,6 @@ def evaluate(
         metric_binds=metric_binds,
         max_concurrency=max_concurrency,
         show_progress_bar=show_progress_bar,
-        auto_download_nltk=auto_download_nltk,
     )
     aggregated_metrics = df_final.mean(numeric_only=True)
     assert isinstance(aggregated_metrics, pd.Series)

{aidial_rag_eval-0.6.0.dev16 → aidial_rag_eval-0.6.0.dev19}/src/aidial_rag_eval/generation/inference.py

@@ -203,7 +203,6 @@ def _segment_hypotheses(
     llm: BaseChatModel,
     max_concurrency: int = 8,
     show_progress_bar: bool = True,
-    auto_download_nltk: bool = True,
 ) -> Tuple[List[SegmentedText], List[Union[SegmentedText, ErrorInfo]]]:
     """
     Function that segments hypotheses into hypothesis segments(roughly into
@@ -236,8 +235,7 @@ def _segment_hypotheses(
         max_concurrency=max_concurrency,
     )
     segmented_hypotheses = [
-        SegmentedText.from_text(text=hypothesis, auto_download_nltk=auto_download_nltk)
-        for hypothesis in hypotheses
+        SegmentedText.from_text(text=hypothesis) for hypothesis in hypotheses
     ]
     if show_progress_bar:
         print("Converting hypothesis...")
@@ -294,14 +292,11 @@ def _extract_statements(
 def _add_questions_to_premises(
     premises: List[Premise],
     questions: Optional[List[Question]],
-    auto_download_nltk: bool,
 ) -> List[Premise]:
     adjusted_premises: List[Premise] = list(premises)
     if questions is not None:
         for i, question in enumerate(questions):
-            question_split = SegmentedText.from_text(
-                text=question, auto_download_nltk=auto_download_nltk
-            )
+            question_split = SegmentedText.from_text(text=question)
             adjusted_premises[i] = question_split.segments[-1] + "\n" + premises[i]
     return adjusted_premises
 
@@ -314,7 +309,6 @@ def _infer_statements(
     list_documents: Optional[List[Documents]] = None,
     max_concurrency: int = 8,
     show_progress_bar: bool = True,
-    auto_download_nltk: bool = True,
 ) -> List[List[Tuple[InferenceInputs, InferenceScore]]]:
     """
     Function that infers statements.
@@ -358,9 +352,7 @@ def _infer_statements(
         For items that failed in any stage, InferenceScore will have inference=None
         and the error field set.
     """
-    adjusted_premises = _add_questions_to_premises(
-        premises, questions, auto_download_nltk
-    )
+    adjusted_premises = _add_questions_to_premises(premises, questions)
 
     document_names: List[JoinedDocumentsName] = (
         [""] * len(statements)
@@ -391,14 +383,12 @@ def segment_hypotheses(
     llm: BaseChatModel,
     max_concurrency: int = 8,
     show_progress_bar: bool = True,
-    auto_download_nltk: bool = True,
 ) -> List[Union[SegmentedText, ErrorInfo]]:
     return _segment_hypotheses(
         hypotheses=hypotheses,
         llm=llm,
         max_concurrency=max_concurrency,
         show_progress_bar=show_progress_bar,
-        auto_download_nltk=auto_download_nltk,
     )[1]
 
 
@@ -426,7 +416,6 @@ def infer_statements(
     list_documents: Optional[List[Documents]] = None,
     max_concurrency: int = 8,
     show_progress_bar: bool = True,
-    auto_download_nltk: bool = True,
 ) -> List[List[Tuple[InferenceInputs, InferenceScore]]]:
     return _infer_statements(
         premises=premises,
@@ -436,7 +425,6 @@ def infer_statements(
         list_documents=list_documents,
         max_concurrency=max_concurrency,
         show_progress_bar=show_progress_bar,
-        auto_download_nltk=auto_download_nltk,
     )
 
 
@@ -484,7 +472,6 @@ def calculate_batch_inference(
     list_documents: Optional[List[Documents]] = None,
     max_concurrency: int = 8,
     show_progress_bar: bool = True,
-    auto_download_nltk: bool = True,
 ) -> List[InferenceReturn]:
     """
     Calculates pairwise the inference of a hypotheses from a premises.
@@ -530,7 +517,6 @@ def calculate_batch_inference(
         llm=llm,
         max_concurrency=max_concurrency,
         show_progress_bar=show_progress_bar,
-        auto_download_nltk=auto_download_nltk,
     )
     statements: List[Union[HypothesisStatements, ErrorInfo]] = _extract_statements(
         segmented_hypotheses=decontextualized_segmented_hypotheses,
@@ -547,7 +533,6 @@ def calculate_batch_inference(
             list_documents=list_documents,
             max_concurrency=max_concurrency,
             show_progress_bar=show_progress_bar,
-            auto_download_nltk=auto_download_nltk,
         )
     )
 
@@ -585,7 +570,6 @@ def calculate_inference(
     documents: Optional[Documents] = None,
     max_concurrency: int = 8,
     show_progress_bar: bool = True,
-    auto_download_nltk: bool = True,
 ) -> InferenceReturn:
     """
     Calculates the inference of a hypothesis from a premise.
@@ -631,6 +615,5 @@ def calculate_inference(
         list_documents=list_documents,
         max_concurrency=max_concurrency,
         show_progress_bar=show_progress_bar,
-        auto_download_nltk=auto_download_nltk,
     )
     return inference_returns[0]

{aidial_rag_eval-0.6.0.dev16 → aidial_rag_eval-0.6.0.dev19}/src/aidial_rag_eval/generation/metric_binds.py

@@ -33,7 +33,6 @@ def _wrapped_dataframe_inference(
     document_column: Optional[str] = None,
     max_concurrency: int = 8,
     show_progress_bar: bool = True,
-    auto_download_nltk: bool = True,
 ) -> pd.DataFrame:
     inference_returns = calculate_batch_inference(
         premises=_get_column_as_list_str(df_merged, premise_column),
@@ -51,7 +50,6 @@ def _wrapped_dataframe_inference(
         ),
         max_concurrency=max_concurrency,
         show_progress_bar=show_progress_bar,
-        auto_download_nltk=auto_download_nltk,
     )
     return pd.DataFrame(
         [dataclasses.asdict(inference_return) for inference_return in inference_returns]
@@ -65,14 +63,12 @@ def _wrapped_dataframe_refusal(
     prefix: str,
     max_concurrency: int = 8,
     show_progress_bar: bool = True,
-    auto_download_nltk: bool = True,
 ) -> pd.DataFrame:
     refusal_returns = calculate_batch_refusal(
         answers=_get_column_as_list_str(df_merged, answer_column),
         llm=llm,
         max_concurrency=max_concurrency,
         show_progress_bar=show_progress_bar,
-        auto_download_nltk=auto_download_nltk,
     )
     return pd.DataFrame(
         [dataclasses.asdict(refusal) for refusal in refusal_returns]
@@ -80,7 +76,7 @@ def _wrapped_dataframe_refusal(
 
 
 def context_to_answer_inference(
-    df_merged, llm, max_concurrency, show_progress_bar, auto_download_nltk, **kwargs
+    df_merged, llm, max_concurrency, show_progress_bar, **kwargs
 ) -> pd.DataFrame:
     return _wrapped_dataframe_inference(
         df_merged=df_merged,
@@ -98,12 +94,11 @@ def context_to_answer_inference(
         document_column=MergedColumns.DOCUMENTS,
         max_concurrency=max_concurrency,
         show_progress_bar=show_progress_bar,
-        auto_download_nltk=auto_download_nltk,
     )
 
 
 def answer_to_ground_truth_inference(
-    df_merged, llm, max_concurrency, show_progress_bar, auto_download_nltk, **kwargs
+    df_merged, llm, max_concurrency, show_progress_bar, **kwargs
 ) -> pd.DataFrame:
     return _wrapped_dataframe_inference(
         df_merged=df_merged,
@@ -115,12 +110,11 @@ def answer_to_ground_truth_inference(
         document_column=MergedColumns.DOCUMENTS,
         max_concurrency=max_concurrency,
         show_progress_bar=show_progress_bar,
-        auto_download_nltk=auto_download_nltk,
     )
 
 
 def ground_truth_to_answer_inference(
-    df_merged, llm, max_concurrency, show_progress_bar, auto_download_nltk, **kwargs
+    df_merged, llm, max_concurrency, show_progress_bar, **kwargs
 ) -> pd.DataFrame:
     return _wrapped_dataframe_inference(
         df_merged=df_merged,
@@ -132,12 +126,11 @@ def ground_truth_to_answer_inference(
         document_column=MergedColumns.DOCUMENTS,
         max_concurrency=max_concurrency,
         show_progress_bar=show_progress_bar,
-        auto_download_nltk=auto_download_nltk,
     )
 
 
 def answer_refusal(
-    df_merged, llm, max_concurrency, show_progress_bar, auto_download_nltk, **kwargs
+    df_merged, llm, max_concurrency, show_progress_bar, **kwargs
 ) -> pd.DataFrame:
     return _wrapped_dataframe_refusal(
         df_merged=df_merged,
@@ -146,12 +139,11 @@ def answer_refusal(
         prefix=ANSWER_REFUSAL_PREFIX,
         max_concurrency=max_concurrency,
         show_progress_bar=show_progress_bar,
-        auto_download_nltk=auto_download_nltk,
     )
 
 
 def ground_truth_refusal(
-    df_merged, llm, max_concurrency, show_progress_bar, auto_download_nltk, **kwargs
+    df_merged, llm, max_concurrency, show_progress_bar, **kwargs
 ) -> pd.DataFrame:
     return _wrapped_dataframe_refusal(
         df_merged=df_merged,
@@ -160,7 +152,6 @@ def ground_truth_refusal(
         prefix=GT_ANSWER_REFUSAL_PREFIX,
         max_concurrency=max_concurrency,
         show_progress_bar=show_progress_bar,
-        auto_download_nltk=auto_download_nltk,
     )
 
 

{aidial_rag_eval-0.6.0.dev16 → aidial_rag_eval-0.6.0.dev19}/src/aidial_rag_eval/generation/refusal.py

@@ -15,7 +15,6 @@ def calculate_batch_refusal(
     llm: BaseChatModel,
     max_concurrency: int = 8,
     show_progress_bar: bool = True,
-    auto_download_nltk: bool = True,
 ) -> List[RefusalReturn]:
     """
     Checks if the answers are answer refusal.
@@ -41,10 +40,7 @@ def calculate_batch_refusal(
         Returns the list of the answer refusals.
     """
     detector = LLMRefusalDetector(llm, max_concurrency)
-    answers_split = [
-        SegmentedText.from_text(text=answer, auto_download_nltk=auto_download_nltk)
-        for answer in answers
-    ]
+    answers_split = [SegmentedText.from_text(text=answer) for answer in answers]
     # As a heuristic, we send only the first 3 segments in the prompt.
     # We believe that if there are 3 whole segments with information
     # that is not related to refusal to answer,
@@ -64,7 +60,6 @@ def calculate_refusal(
     llm: BaseChatModel,
     max_concurrency: int = 8,
     show_progress_bar: bool = True,
-    auto_download_nltk: bool = True,
 ) -> RefusalReturn:
     """
     Checks if the answer is answer refusal.
@@ -94,6 +89,5 @@ def calculate_refusal(
         llm=llm,
         max_concurrency=max_concurrency,
         show_progress_bar=show_progress_bar,
-        auto_download_nltk=auto_download_nltk,
     )
     return refusal_returns[0]

{aidial_rag_eval-0.6.0.dev16 → aidial_rag_eval-0.6.0.dev19}/src/aidial_rag_eval/generation/utils/segmented_text.py

@@ -1,11 +1,32 @@
 from typing import Callable, List, Optional, Tuple
 
-import nltk
-from more_itertools import chunked
+import spacy
+from more_itertools import chunked, constrained_batches
 
 from aidial_rag_eval.generation.types import Text, TextSegment
 
 Splitter = Callable[[TextSegment], List[TextSegment]]
+
+_nlp = spacy.load("en_core_web_sm")
+
+_simple_nlp = spacy.blank("en")
+_simple_nlp.add_pipe("sentencizer")
+_simple_nlp.max_length = 1_000_000_000
+
+
+def _sent_tokenize(text: str) -> List[str]:
+    max_len = _nlp.max_length
+    sents = list(_simple_nlp(text).sents)
+
+    chunks = [
+        "".join(s.text_with_ws for s in batch)
+        for batch in constrained_batches(sents, max_len, get_len=lambda s: len(s.text))
+    ]
+    return [
+        s.text.strip() for chunk in chunks for s in _nlp(chunk).sents if s.text.strip()
+    ]
+
+
 SegmentChecker = Callable[[TextSegment], bool]
 
 
@@ -80,10 +101,7 @@ class SegmentedText:
         self.delimiters = delimiters.copy()
 
     @classmethod
-    def from_text(cls, text: Text, auto_download_nltk: bool = True) -> "SegmentedText":
-        if auto_download_nltk:
-            nltk.download("punkt_tab", quiet=True)
-
+    def from_text(cls, text: Text) -> "SegmentedText":
         max_len = 500
         min_len = 10
         conditional_splitters: List[Splitter] = [
@@ -93,7 +111,7 @@ class SegmentedText:
         ]
         segmented_text = cls([text], [])
         segmented_text = apply_splitter_to_segmented_text(
-            segmented_text, nltk.sent_tokenize
+            segmented_text, _sent_tokenize
         )
         for splitter in conditional_splitters:
             segmented_text = apply_splitter_to_segmented_text(