rasa-pro 3.12.20__tar.gz → 3.12.21__tar.gz

{rasa_pro-3.12.20 → rasa_pro-3.12.21}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.3
 Name: rasa-pro
-Version: 3.12.20
+Version: 3.12.21
 Summary: State-of-the-art open-core Conversational AI framework for Enterprises that natively leverages generative AI for effortless assistant development.
 Keywords: nlp,machine-learning,machine-learning-library,bot,bots,botkit,rasa conversational-agents,conversational-ai,chatbot,chatbot-framework,bot-framework
 Author: Rasa Technologies GmbH

{rasa_pro-3.12.20 → rasa_pro-3.12.21}/pyproject.toml

@@ -4,7 +4,7 @@ build-backend = "poetry.core.masonry.api"
 
 [tool.poetry]
 name = "rasa-pro"
-version = "3.12.20"
+version = "3.12.21"
 description = "State-of-the-art open-core Conversational AI framework for Enterprises that natively leverages generative AI for effortless assistant development."
 authors = [ "Rasa Technologies GmbH <hi@rasa.com>",]
 maintainers = [ "Tom Bocklisch <tom@rasa.com>",]
@@ -375,6 +375,7 @@ convention = "google"
 
 
 
+
 [tool.poetry.group.dev.dependencies]
 ruff = ">=0.3.5,<0.4.0"
 docker = "~7.1.0"

{rasa_pro-3.12.20 → rasa_pro-3.12.21}/rasa/core/policies/enterprise_search_policy.py

@@ -1,7 +1,9 @@
+import glob
 import importlib.resources
 import json
+import os.path
 import re
-from typing import TYPE_CHECKING, Any, Dict, List, Optional, Text
+from typing import TYPE_CHECKING, Any, Dict, List, Optional, Text, Tuple
 
 import dotenv
 import structlog
@@ -162,6 +164,8 @@ DEFAULT_ENTERPRISE_SEARCH_PROMPT_WITH_CITATION_TEMPLATE = importlib.resources.re
     "rasa.core.policies", "enterprise_search_prompt_with_citation_template.jinja2"
 )
 
+_ENTERPRISE_SEARCH_CITATION_PATTERN = re.compile(r"\[([^\]]+)\]")
+
 
 class VectorStoreConnectionError(RasaException):
     """Exception raised for errors in connecting to the vector store."""
@@ -378,9 +382,11 @@ class EnterpriseSearchPolicy(LLMHealthCheckMixin, EmbeddingsHealthCheckMixin, Po
 
         if store_type == DEFAULT_VECTOR_STORE_TYPE:
             logger.info("enterprise_search_policy.train.faiss")
+            docs_folder = self.vector_store_config.get(SOURCE_PROPERTY)
+            self._validate_documents_folder(docs_folder)
             with self._model_storage.write_to(self._resource) as path:
                 self.vector_store = FAISS_Store(
-                    docs_folder=self.vector_store_config.get(SOURCE_PROPERTY),
+                    docs_folder=docs_folder,
                     embeddings=embeddings,
                     index_path=path,
                     create_index=True,
@@ -760,6 +766,33 @@ class EnterpriseSearchPolicy(LLMHealthCheckMixin, EmbeddingsHealthCheckMixin, Po
             result[domain.index_for_action(action_name)] = score  # type: ignore[assignment]
         return result
 
+    @classmethod
+    def _validate_documents_folder(cls, docs_folder: str) -> None:
+        if not os.path.exists(docs_folder) or not os.path.isdir(docs_folder):
+            error_message = (
+                f"Document source directory does not exist or is not a "
+                f"directory: '{docs_folder}'. "
+                "Please specify a valid path to the documents source directory in the "
+                "vector_store configuration."
+            )
+            logger.error(
+                "enterprise_search_policy.train.faiss.invalid_source_directory",
+                message=error_message,
+            )
+            print_error_and_exit(error_message)
+
+        docs = glob.glob(os.path.join(docs_folder, "*.txt"), recursive=True)
+        if not docs or len(docs) < 1:
+            error_message = (
+                f"Document source directory is empty: '{docs_folder}'. "
+                "Please add documents to this directory or specify a different one."
+            )
+            logger.error(
+                "enterprise_search_policy.train.faiss.source_directory_empty",
+                message=error_message,
+            )
+            print_error_and_exit(error_message)
+
     @classmethod
     def load(
         cls,
@@ -833,7 +866,7 @@ class EnterpriseSearchPolicy(LLMHealthCheckMixin, EmbeddingsHealthCheckMixin, Po
             return None
 
         source = merged_config.get(VECTOR_STORE_PROPERTY, {}).get(SOURCE_PROPERTY)
-        if not source:
+        if not source or not os.path.exists(source) or not os.path.isdir(source):
             return None
 
         docs = FAISS_Store.load_documents(source)
@@ -870,10 +903,18 @@ class EnterpriseSearchPolicy(LLMHealthCheckMixin, EmbeddingsHealthCheckMixin, Po
 
     @staticmethod
     def post_process_citations(llm_answer: str) -> str:
-        """Post-process the LLM answer.
-
-         Re-writes the bracketed numbers to start from 1 and
-         re-arranges the sources to follow the enumeration order.
+        """Post-processes the LLM answer to correctly number and sort citations and
+        sources.
+
+        - Handles both single `[1]` and grouped `[1, 3]` citations.
+        - Rewrites the numbers in square brackets in the answer text to start from 1
+        and be sorted within each group.
+        - Reorders the sources according to the order of their first appearance
+        in the text.
+        - Removes citations from the text that point to sources missing from
+        the source list.
+        - Keeps sources that are not cited in the text, placing them at the end
+        of the list.
 
         Args:
             llm_answer: The LLM answer.
@@ -887,77 +928,160 @@ class EnterpriseSearchPolicy(LLMHealthCheckMixin, EmbeddingsHealthCheckMixin, Po
 
         # Split llm_answer into answer and citations
         try:
-            answer, citations = llm_answer.rsplit("Sources:", 1)
+            answer_part, sources_part = llm_answer.rsplit("Sources:", 1)
         except ValueError:
-            # if there is no "Sources:" in the llm_answer
-            return llm_answer
-
-        # Find all source references in the answer
-        pattern = r"\[\s*(\d+(?:\s*,\s*\d+)*)\s*\]"
-        matches = re.findall(pattern, answer)
-        old_source_indices = [
-            int(num.strip()) for match in matches for num in match.split(",")
-        ]
+            # if there is no "Sources:" separator, return the original llm_answer
+            return llm_answer.strip()
+
+        # Parse the sources block to extract valid sources and other lines
+        valid_sources, other_source_lines = EnterpriseSearchPolicy._parse_sources_block(
+            sources_part
+        )
+
+        # Find all unique, valid citations in the answer text in their order
+        # of appearance
+        cited_order = EnterpriseSearchPolicy._get_cited_order(
+            answer_part, valid_sources
+        )
+
+        # Create a mapping from the old source numbers to the new, sequential numbers.
+        # For example, if the citation order in the text was [3, 1, 2], this map
+        # becomes {3: 1, 1: 2, 2: 3}. This allows for a quick lookup when rewriting
+        # the citations
+        renumbering_map = {
+            old_num: new_num + 1 for new_num, old_num in enumerate(cited_order)
+        }
+
+        # Rewrite the citations in the answer text based on the renumbering map
+        processed_answer = EnterpriseSearchPolicy._rewrite_answer_citations(
+            answer_part, renumbering_map
+        )
+
+        # Build the new list of sources
+        new_sources_list = EnterpriseSearchPolicy._build_final_sources_list(
+            cited_order,
+            renumbering_map,
+            valid_sources,
+            other_source_lines,
+        )
+
+        if len(new_sources_list) > 0:
+            processed_answer += "\nSources:\n" + "\n".join(new_sources_list)
 
-        # Map old source references to the correct enumeration
-        renumber_mapping = {num: idx + 1 for idx, num in enumerate(old_source_indices)}
-
-        # remove whitespace from original source citations in answer
-        for match in matches:
-            answer = answer.replace(f"[{match}]", f"[{match.replace(' ', '')}]")
-
-        new_answer = []
-        for word in answer.split():
-            matches = re.findall(pattern, word)
-            if matches:
-                for match in matches:
-                    if "," in match:
-                        old_indices = [
-                            int(num.strip()) for num in match.split(",") if num
-                        ]
-                        new_indices = [
-                            renumber_mapping[old_index]
-                            for old_index in old_indices
-                            if old_index in renumber_mapping
-                        ]
-                        if not new_indices:
-                            continue
-
-                        word = word.replace(
-                            match, f"{', '.join(map(str, new_indices))}"
-                        )
-                    else:
-                        old_index = int(match.strip("[].,:;?!"))
-                        new_index = renumber_mapping.get(old_index)
-                        if not new_index:
-                            continue
-
-                        word = word.replace(str(old_index), str(new_index))
-            new_answer.append(word)
-
-        # join the words
-        joined_answer = " ".join(new_answer)
-        joined_answer += "\nSources:\n"
-
-        new_sources: List[str] = []
-
-        for line in citations.split("\n"):
-            pattern = r"(?<=\[)\d+"
-            match = re.search(pattern, line)
+        return processed_answer
+
+    @staticmethod
+    def _parse_sources_block(sources_part: str) -> Tuple[Dict[int, str], List[str]]:
+        """Parses the sources block from the LLM response.
+        Returns a tuple containing:
+        - A dictionary of valid sources matching the "[1] ..." format,
+        where the key is the source number
+        - A list of other source lines that do not match the specified format
+        """
+        valid_sources: Dict[int, str] = {}
+        other_source_lines: List[str] = []
+        source_line_pattern = re.compile(r"^\s*\[(\d+)\](.*)")
+
+        source_lines = sources_part.strip().split("\n")
+
+        for line in source_lines:
+            line = line.strip()
+            if not line:
+                continue
+
+            match = source_line_pattern.match(line)
             if match:
-                old_index = int(match.group(0))
-                new_index = renumber_mapping[old_index]
-                # replace only the first occurrence of the old index
-                line = line.replace(f"[{old_index}]", f"[{new_index}]", 1)
+                num = int(match.group(1))
+                valid_sources[num] = line
+            else:
+                other_source_lines.append(line)
+
+        return valid_sources, other_source_lines
+
+    @staticmethod
+    def _get_cited_order(
+        answer_part: str, available_sources: Dict[int, str]
+    ) -> List[int]:
+        """Find all unique, valid citations in the answer text in their order
+        # of appearance
+        """
+        cited_order: List[int] = []
+        seen_indices = set()
+
+        for match in _ENTERPRISE_SEARCH_CITATION_PATTERN.finditer(answer_part):
+            content = match.group(1)
+            indices_str = [s.strip() for s in content.split(",")]
+            for index_str in indices_str:
+                if index_str.isdigit():
+                    index = int(index_str)
+                    if index in available_sources and index not in seen_indices:
+                        cited_order.append(index)
+                        seen_indices.add(index)
+
+        return cited_order
+
+    @staticmethod
+    def _rewrite_answer_citations(
+        answer_part: str, renumber_map: Dict[int, int]
+    ) -> str:
+        """Rewrites the citations in the answer text based on the renumbering map."""
+
+        def replacer(match: re.Match) -> str:
+            content = match.group(1)
+            old_indices_str = [s.strip() for s in content.split(",")]
+            new_indices = [
+                renumber_map[int(s)]
+                for s in old_indices_str
+                if s.isdigit() and int(s) in renumber_map
+            ]
+            if not new_indices:
+                return ""
+
+            return f"[{', '.join(map(str, sorted(list(set(new_indices)))))}]"
+
+        processed_answer = _ENTERPRISE_SEARCH_CITATION_PATTERN.sub(
+            replacer, answer_part
+        )
+
+        # Clean up formatting after replacements
+        processed_answer = re.sub(r"\s+([,.?])", r"\1", processed_answer)
+        processed_answer = processed_answer.replace("[]", " ")
+        processed_answer = re.sub(r"\s+", " ", processed_answer)
+        processed_answer = processed_answer.strip()
+
+        return processed_answer
+
+    @staticmethod
+    def _build_final_sources_list(
+        cited_order: List[int],
+        renumbering_map: Dict[int, int],
+        valid_sources: Dict[int, str],
+        other_source_lines: List[str],
+    ) -> List[str]:
+        """Builds the final list of sources based on the cited order and
+        renumbering map.
+        """
+        new_sources_list: List[str] = []
+
+        # First, add the sorted, used sources
+        for old_num in cited_order:
+            new_num = renumbering_map[old_num]
+            source_line = valid_sources[old_num]
+            new_sources_list.append(
+                source_line.replace(f"[{old_num}]", f"[{new_num}]", 1)
+            )
 
-                # insert the line into the new_index position
-                new_sources.insert(new_index - 1, line)
-            elif line.strip():
-                new_sources.append(line)
+        # Then, add the unused but validly numbered sources
+        used_source_nums = set(cited_order)
+        # Sort by number to ensure a consistent order for uncited sources
+        for num, line in sorted(valid_sources.items()):
+            if num not in used_source_nums:
+                new_sources_list.append(line)
 
-        joined_sources = "\n".join(new_sources)
+        # Finally, add any other source lines
+        new_sources_list.extend(other_source_lines)
 
-        return joined_answer + joined_sources
+        return new_sources_list
 
     @classmethod
     def _perform_health_checks(

{rasa_pro-3.12.20 → rasa_pro-3.12.21}/rasa/dialogue_understanding/processor/command_processor.py

@@ -64,12 +64,6 @@ from rasa.shared.nlu.constants import COMMANDS
 
 structlogger = structlog.get_logger()
 
-CANNOT_HANDLE_REASON = (
-    "A command generator attempted to set a slot "
-    "with a value extracted by an extractor "
-    "that is incompatible with the slot mapping type."
-)
-
 
 def contains_command(commands: List[Command], typ: Type[Command]) -> bool:
     """Check if a list of commands contains a command of a given type.
@@ -588,6 +582,11 @@ def clean_up_slot_command(
             "command_processor.clean_up_slot_command.skip_command_slot_not_in_domain",
             command=command,
         )
+        resulting_commands.append(
+            CannotHandleCommand(
+                reason="The slot predicted by the LLM is not defined in the domain."
+            )
+        )
         return resulting_commands
 
     if not should_slot_be_set(slot, command, resulting_commands):
@@ -606,7 +605,10 @@ def clean_up_slot_command(
             for command in resulting_commands
         )
 
-        cannot_handle = CannotHandleCommand(reason=CANNOT_HANDLE_REASON)
+        cannot_handle = CannotHandleCommand(
+            reason="A command generator attempted to set a slot with a value extracted "
+            "by an extractor that is incompatible with the slot mapping type."
+        )
         if not slot_command_exists_already and cannot_handle not in resulting_commands:
             resulting_commands.append(cannot_handle)
 
@@ -640,9 +642,9 @@ def clean_up_slot_command(
             resulting_commands.append(command)
             return resulting_commands
 
-        if (slot := tracker.slots.get(command.name)) is not None and slot.value == str(
-            command.value
-        ):
+        if (slot := tracker.slots.get(command.name)) is not None and str(
+            slot.value
+        ) == str(command.value):
             # the slot is already set, we don't need to set it again
             structlogger.debug(
                 "command_processor.clean_up_slot_command.skip_command_slot_already_set",

{rasa_pro-3.12.20 → rasa_pro-3.12.21}/rasa/llm_fine_tuning/annotation_module.py

@@ -9,8 +9,8 @@ from rasa.e2e_test.e2e_test_case import ActualStepOutput, TestCase, TestStep, Te
 from rasa.e2e_test.e2e_test_runner import TEST_TURNS_TYPE, E2ETestRunner
 from rasa.llm_fine_tuning.conversations import Conversation, ConversationStep
 from rasa.llm_fine_tuning.storage import StorageContext
-from rasa.shared.core.constants import USER
-from rasa.shared.core.events import UserUttered
+from rasa.shared.core.constants import BOT, USER
+from rasa.shared.core.events import BotUttered, UserUttered
 from rasa.shared.core.trackers import DialogueStateTracker
 from rasa.shared.exceptions import FinetuningDataPreparationException
 from rasa.shared.nlu.constants import LLM_COMMANDS, LLM_PROMPT
@@ -83,16 +83,18 @@ def generate_conversation(
         Conversation.
     """
     steps = []
-    tracker_event_indices = [
-        i for i, event in enumerate(tracker.events) if isinstance(event, UserUttered)
-    ]
-
-    if len(test_case.steps) != len(tracker_event_indices):
-        raise FinetuningDataPreparationException(
-            "Number of test case steps and tracker events do not match."
-        )
 
     if assertions_used:
+        tracker_event_indices = [
+            i
+            for i, event in enumerate(tracker.events)
+            if isinstance(event, UserUttered)
+        ]
+        if len(test_case.steps) != len(tracker_event_indices):
+            raise FinetuningDataPreparationException(
+                "Number of test case steps and tracker events do not match."
+            )
+
         # we only have user steps, extract the bot response from the bot uttered
         # events of the test turn
         for i, (original_step, tracker_event_index) in enumerate(
@@ -110,8 +112,30 @@ def generate_conversation(
             )
             steps.extend(_create_bot_test_steps(test_turns[i]))
     else:
+        tracker_event_indices = [
+            i
+            for i, event in enumerate(tracker.events)
+            if isinstance(event, UserUttered) or isinstance(event, BotUttered)
+        ]
+
+        # Generally, we expect one or more bot response(s) for each user utterance
+        # in the test case, so that we can evaluate the actual bot response.
+        # If the test case ends with one or more user utterance(s) instead,
+        # we should thus trim those from the test case steps.
+        # This only applies to test cases that have at least one bot utterance;
+        # otherwise, all test case steps would be removed.
+        has_bot_utterance = any(step.actor == BOT for step in test_case.steps)
+        i = len(test_case.steps)
+        if has_bot_utterance:
+            while i > 0 and test_case.steps[i - 1].actor == USER:
+                i -= 1
+        test_case_steps = test_case.steps[:i]
+
+        # If the number of test case steps and tracker events differ,
+        # using zip ensures we only process pairs that exist in both lists.
+        # Prevents index errors and ensures we don't process unmatched steps or events.
         for i, (original_step, tracker_event_index) in enumerate(
-            zip(test_case.steps, tracker_event_indices)
+            zip(test_case_steps, tracker_event_indices)
         ):
             if original_step.actor == USER:
                 previous_turn = _get_previous_actual_step_output(test_turns, i)
@@ -127,6 +151,14 @@ def generate_conversation(
             else:
                 steps.append(original_step)
 
+        # the tracker should only include events up to the last bot utterance
+        # so that the resulting transcript ends with the last bot utterance too
+        # only applies to test cases that have at least one bot utterance
+        if has_bot_utterance and test_case.steps and test_case.steps[-1].actor == USER:
+            event_to_go_to = tracker_event_indices[len(test_case_steps)] - 1
+            timestamp = tracker.events[event_to_go_to].timestamp
+            tracker = tracker.travel_back_in_time(timestamp)
+
     # Some messages in an e2e test case could be mapped to commands via
     # 'NLUCommandAdapter', e.g. the message will not be annotated with a prompt and
     # commands pair. Only convert steps that have a prompt and commands present into a

{rasa_pro-3.12.20 → rasa_pro-3.12.21}/rasa/shared/core/constants.py

@@ -181,6 +181,7 @@ class SetSlotExtractor(Enum):
 # the keys for `State` (USER, PREVIOUS_ACTION, SLOTS, ACTIVE_LOOP)
 # represent the origin of a `SubState`
 USER = "user"
+BOT = "bot"
 SLOTS = "slots"
 
 USE_TEXT_FOR_FEATURIZATION = "use_text_for_featurization"

{rasa_pro-3.12.20 → rasa_pro-3.12.21}/rasa/version.py

@@ -1,3 +1,3 @@
 # this file will automatically be changed,
 # do not add anything but the version number here!
-__version__ = "3.12.20"
+__version__ = "3.12.21"