rasa-pro 3.12.22__py3-none-any.whl → 3.13.0__py3-none-any.whl

rasa/core/concurrent_lock_store.py

@@ -1,9 +1,10 @@
 import json
-import logging
 import time
 from collections import deque
 from typing import Deque, Optional, Text
 
+import structlog
+
 from rasa.core.lock import Ticket, TicketLock
 from rasa.core.lock_store import (
     DEFAULT_SOCKET_TIMEOUT_IN_SECONDS,
@@ -19,7 +20,7 @@ DEFAULT_PORT = 6379
 
 DEFAULT_HOSTNAME = "localhost"
 
-logger = logging.getLogger(__name__)
+structlogger = structlog.getLogger(__name__)
 
 LAST_ISSUED_TICKET_NUMBER_SUFFIX = "last_issued_ticket_number"
 
@@ -105,7 +106,10 @@ class ConcurrentRedisLockStore(LockStore):
 
         self.key_prefix = DEFAULT_CONCURRENT_REDIS_LOCK_STORE_KEY_PREFIX
         if key_prefix:
-            logger.debug(f"Setting non-default redis key prefix: '{key_prefix}'.")
+            structlogger.debug(
+                "concurrent_redis_lock_store._set_key_prefix.non_default_key_prefix",
+                event_info=f"Setting non-default redis key prefix: '{key_prefix}'.",
+            )
             self._set_key_prefix(key_prefix)
 
         super().__init__()
@@ -116,9 +120,13 @@ class ConcurrentRedisLockStore(LockStore):
                 key_prefix + ":" + DEFAULT_CONCURRENT_REDIS_LOCK_STORE_KEY_PREFIX
             )
         else:
-            logger.warning(
-                f"Omitting provided non-alphanumeric redis key prefix: '{key_prefix}'. "
-                f"Using default '{self.key_prefix}' instead."
+            structlogger.warning(
+                "concurrent_redis_lock_store._set_key_prefix.default_instead_of_invalid_key_prefix",
+                event_info=(
+                    f"Omitting provided non-alphanumeric "
+                    f"redis key prefix: '{key_prefix}'. "
+                    f"Using default '{self.key_prefix}' instead."
+                ),
             )
 
     def issue_ticket(
@@ -129,7 +137,10 @@ class ConcurrentRedisLockStore(LockStore):
         It's configured with `lock_lifetime` and associated with `conversation_id`.
         Creates a new lock if none is found.
         """
-        logger.debug(f"Issuing ticket for conversation '{conversation_id}'.")
+        structlogger.debug(
+            "concurrent_redis_lock_store.issue_ticket",
+            event_info=f"Issuing ticket for conversation '{conversation_id}'.",
+        )
         try:
             lock = self.get_or_create_lock(conversation_id)
             lock.remove_expired_tickets()
@@ -164,9 +175,12 @@ class ConcurrentRedisLockStore(LockStore):
         redis_keys = self.red.keys(pattern)
 
         if not redis_keys:
-            logger.debug(
-                f"The lock store does not contain any key-value "
-                f"items for conversation '{conversation_id}'."
+            structlogger.debug(
+                "concurrent_redis_lock_store.delete_lock_key_not_found",
+                event_info=(
+                    f"The lock store does not contain any key-value "
+                    f"items for conversation '{conversation_id}'."
+                ),
             )
             return None
 

rasa/core/evaluation/marker_tracker_loader.py

@@ -2,7 +2,7 @@ import random
 from typing import Any, AsyncIterator, Iterable, List, Optional, Text
 
 import rasa.shared.utils.io
-from rasa.core.tracker_store import TrackerStore
+from rasa.core.tracker_stores.tracker_store import TrackerStore
 from rasa.shared.core.trackers import DialogueStateTracker
 from rasa.shared.exceptions import RasaException
 

rasa/core/exporter.py

@@ -10,12 +10,17 @@ import rasa.shared.utils.io
 from rasa.core.brokers.broker import EventBroker
 from rasa.core.brokers.pika import PikaEventBroker
 from rasa.core.constants import RASA_EXPORT_PROCESS_ID_HEADER_NAME
-from rasa.core.tracker_store import TrackerStore
+from rasa.core.tracker_stores.tracker_store import TrackerStore
 from rasa.exceptions import (
     NoConversationsInTrackerStoreError,
     NoEventsToMigrateError,
     PublishingError,
 )
+from rasa.shared.core.events import (
+    BotUttered,
+    SlotSet,
+    UserUttered,
+)
 from rasa.shared.core.trackers import EventVerbosity
 
 logger = logging.getLogger(__name__)
@@ -43,6 +48,7 @@ class Exporter:
         tracker_store: TrackerStore,
         event_broker: EventBroker,
         endpoints_path: Text,
+        is_pii_enabled: bool = False,
         requested_conversation_ids: Optional[Text] = None,
         minimum_timestamp: Optional[float] = None,
         maximum_timestamp: Optional[float] = None,
@@ -52,6 +58,7 @@ class Exporter:
         self.tracker_store = tracker_store
 
         self.event_broker = event_broker
+        self.is_pii_enabled = is_pii_enabled
         self.requested_conversation_ids = requested_conversation_ids
         self.minimum_timestamp = minimum_timestamp
         self.maximum_timestamp = maximum_timestamp
@@ -72,10 +79,12 @@ class Exporter:
         current_timestamp = None
 
         headers = self._get_message_headers()
+        warned_sender_ids: Set[Text] = set()
 
         async for event in self._fetch_events_within_time_range():
             # noinspection PyBroadException
             try:
+                self._check_anonymization_status(event, warned_sender_ids)
                 self._publish_with_message_headers(event, headers)
                 published_events += 1
                 current_timestamp = event["timestamp"]
@@ -282,3 +291,30 @@ class Exporter:
             events_with_conversation_id.append(event)
 
         return events_with_conversation_id
+
+    def _check_anonymization_status(
+        self, event: Dict[Text, Any], warned_sender_ids: Set[Text]
+    ) -> None:
+        """Check if the tracker store contains unanonymized events.
+
+        If it does, print a warning that these events will be published as is.
+
+        Args:
+            event: The event to check for anonymization status
+            warned_sender_ids: Set of sender IDs that have already been warned about
+        """
+        sender_id = event["sender_id"]
+        if (
+            self.is_pii_enabled
+            and sender_id not in warned_sender_ids
+            and event["event"]
+            in (UserUttered.type_name, BotUttered.type_name, SlotSet.type_name)
+            and not event.get("anonymized_at", None)
+        ):
+            rasa.shared.utils.cli.print_warning(
+                f"Retrieved un-anonymized event for sender_id {sender_id}. "
+                f"All events after this timestamp {event['timestamp']} "
+                "are not anonymized for this tracker. Proceeding with "
+                "publishing plaintext values for all events following this.",
+            )
+            warned_sender_ids.add(sender_id)

rasa/core/http_interpreter.py

@@ -1,4 +1,3 @@
-import copy
 import logging
 from typing import Any, Dict, Optional, Text
 
@@ -49,7 +48,6 @@ class RasaNLUHttpInterpreter:
         if not self.endpoint_config or self.endpoint_config.url is None:
             structlogger.error(
                 "http.parse.text",
-                text=copy.deepcopy(text),
                 event_info="No rasa NLU server specified!",
             )
             return None
@@ -71,18 +69,16 @@ class RasaNLUHttpInterpreter:
                 if resp.status == 200:
                     return await resp.json()
                 else:
-                    response_text = await resp.text()
                     structlogger.error(
                         "http.parse.text.failure",
-                        text=copy.deepcopy(text),
-                        response_text=copy.deepcopy(response_text),
+                        event_info="Failed to parse text",
                     )
                     return None
-        except Exception:  # skipcq: PYL-W0703
+        except Exception as e:  # skipcq: PYL-W0703
             # need to catch all possible exceptions when doing http requests
             # (timeouts, value errors, parser errors, ...)
             structlogger.exception(
                 "http.parse.text.exception",
-                text=copy.deepcopy(text),
+                event_info=f"Exception occurred while parsing text. Error: {e}",
             )
             return None

rasa/core/information_retrieval/faiss.py

@@ -12,6 +12,7 @@ from rasa.core.information_retrieval import (
     InformationRetrievalException,
     SearchResultList,
 )
+from rasa.core.information_retrieval.ingestion.faq_parser import _format_faq_documents
 from rasa.utils.endpoints import EndpointConfig
 from rasa.utils.ml_utils import persist_faiss_vector_store
 
@@ -31,10 +32,12 @@ class FAISS_Store(InformationRetrieval):
         index_path: str,
         docs_folder: Optional[str],
         create_index: Optional[bool] = False,
+        parse_as_faq_pairs: Optional[bool] = False,
     ):
         """Initializes the FAISS Store."""
         self.chunk_size = 1000
         self.chunk_overlap = 20
+        self.parse_as_faq_pairs = parse_as_faq_pairs
 
         path = Path(index_path) / "documents_faiss"
         if create_index:
@@ -86,21 +89,25 @@ class FAISS_Store(InformationRetrieval):
         if not docs_folder:
             raise ValueError("parameter `docs_folder` needs to be specified")
 
-        docs = self.load_documents(docs_folder)
-        splitter = RecursiveCharacterTextSplitter(
-            chunk_size=self.chunk_size,
-            chunk_overlap=self.chunk_overlap,
-            length_function=len,
-        )
-        doc_chunks = splitter.split_documents(docs)
+        documents = self.load_documents(docs_folder)
+
+        if not self.parse_as_faq_pairs:
+            splitter = RecursiveCharacterTextSplitter(
+                chunk_size=self.chunk_size,
+                chunk_overlap=self.chunk_overlap,
+                length_function=len,
+            )
+            parsed_documents = splitter.split_documents(documents)
+        else:
+            parsed_documents = _format_faq_documents(documents)
 
         logger.info(
             "information_retrieval.faiss_store._create_document_index",
-            len_chunks=len(doc_chunks),
+            len_chunks=len(parsed_documents),
         )
-        if doc_chunks:
-            texts = [chunk.page_content for chunk in doc_chunks]
-            metadatas = [chunk.metadata for chunk in doc_chunks]
+        if parsed_documents:
+            texts = [document.page_content for document in parsed_documents]
+            metadatas = [document.metadata for document in parsed_documents]
             return FAISS.from_texts(texts, embedding, metadatas=metadatas, ids=None)
         else:
             raise ValueError(f"No documents found at '{docs_folder}'.")

rasa/core/information_retrieval/ingestion/faq_parser.py

@@ -0,0 +1,158 @@
+"""Utilities for parsing FAQ-style documents (Q/A pairs) used in extractive search."""
+
+import re
+from collections import defaultdict
+from typing import TYPE_CHECKING, List
+
+import structlog
+
+from rasa.shared.constants import (
+    DOCUMENT_TYPE_FAQ,
+    FAQ_DOCUMENT_ENTRY_SEPARATOR,
+    FAQ_DOCUMENT_LINE_SEPARATOR,
+    FAQ_DOCUMENT_METADATA_ANSWER,
+    FAQ_DOCUMENT_METADATA_TITLE,
+    FAQ_DOCUMENT_METADATA_TYPE,
+    FAQ_INPUT_DATA_ANSWER_LINE_PREFIX,
+    FAQ_INPUT_DATA_QUESTION_LINE_PREFIX,
+)
+
+if TYPE_CHECKING:
+    from langchain.schema import Document
+
+_FAQ_PAIR_PATTERN = re.compile(
+    rf"{re.escape(FAQ_INPUT_DATA_QUESTION_LINE_PREFIX)}\s*"
+    rf"(?P<question>.*?)\s*{FAQ_DOCUMENT_LINE_SEPARATOR}\s*"
+    rf"{re.escape(FAQ_INPUT_DATA_ANSWER_LINE_PREFIX)}\s*"
+    rf"(?P<answer>.*)",
+    re.DOTALL,
+)
+
+
+structlogger = structlog.get_logger()
+
+
+def _format_faq_documents(documents: List["Document"]) -> List["Document"]:
+    """Splits each loaded file into individual FAQs.
+
+    Args:
+        documents: Documents representing whole files containing FAQs.
+
+    Returns:
+        List of Document objects, each containing a separate FAQ.
+
+    Examples:
+        An example of a file containing FAQs:
+
+        Q: Who is Finley?
+        A: Finley is your smart assistant for the FinX App. You can add him to your
+           favorite messenger and tell him what you need help with.
+
+        Q: How does Finley work?
+        A: Finley is powered by the latest chatbot technology leveraging a unique
+           interplay of large language models and secure logic.
+
+    More details in documentation: https://rasa.com/docs/reference/config/policies/extractive-search/
+    """
+    structured_faqs = []
+    from langchain.schema import Document
+
+    for document in documents:
+        chunks = document.page_content.strip().split(FAQ_DOCUMENT_ENTRY_SEPARATOR)
+
+        for chunk in chunks:
+            match = _FAQ_PAIR_PATTERN.match(chunk.strip())
+
+            if not match:
+                structlogger.warning(
+                    "faq_parser.format_faq_documents.invalid_chunk_skipped",
+                    event_info=(
+                        "Chunk does not match expected QA format. "
+                        "Please refer to the documentation: "
+                        "https://rasa.com/docs/reference/config/"
+                        "policies/extractive-search/"
+                    ),
+                    chunk_preview=chunk[:100],
+                )
+                continue
+
+            question = match.group("question").strip()
+            answer = match.group("answer").strip()
+            title = _sanitize_title(question)
+
+            formatted_document = Document(
+                page_content=question,
+                metadata={
+                    FAQ_DOCUMENT_METADATA_TITLE: title,
+                    FAQ_DOCUMENT_METADATA_TYPE: DOCUMENT_TYPE_FAQ,
+                    FAQ_DOCUMENT_METADATA_ANSWER: answer,
+                },
+            )
+
+            structured_faqs.append(formatted_document)
+
+            structlogger.debug(
+                "faq_parser.format_faq_documents.parsed_chunk",
+                event_info="Parsed chunk.",
+                title=title,
+                question=question,
+                answer=answer,
+                parsed_chunk_preview=chunk[:100],
+            )
+
+    structlogger.debug(
+        "faq_parser.format_faq_documents.parsed_chunks",
+        event_info=(
+            f"Retrieved {len(structured_faqs)} FAQ pair(s)"
+            f"from {len(documents)} document(s)."
+        ),
+        num_structured_faqs=len(structured_faqs),
+        num_documents=len(documents),
+    )
+    _check_and_parsed_faq_documents_for_duplicates(structured_faqs)
+    return structured_faqs
+
+
+def _sanitize_title(title: str) -> str:
+    title = title.lower()
+    # Remove all whitespaces with "_"
+    title = re.sub(r"\s+", "_", title)
+    # Remove all non alpha-numeric characters
+    title = re.sub(r"[^\w]", "", title)
+    # Collapse multiple "_"
+    title = re.sub(r"_+", "_", title)
+    # Clean up edges
+    return title.strip("_")
+
+
+def _check_and_parsed_faq_documents_for_duplicates(documents: List["Document"]) -> None:
+    seen_qa_pairs = set()
+    seen_questions: defaultdict = defaultdict(list)
+
+    for doc in documents:
+        question = doc.page_content.strip()
+        answer = doc.metadata.get(FAQ_DOCUMENT_METADATA_ANSWER, "").strip()
+
+        if not question or not answer:
+            continue
+
+        if (question, answer) in seen_qa_pairs:
+            structlogger.warning(
+                "faq_parser.duplicate_qa_pair_found",
+                event_info="Duplicate QA pair found.",
+                question=question,
+                answer_preview=answer,
+            )
+            continue
+
+        if question in seen_questions and seen_questions[question] != answer:
+            structlogger.warning(
+                "faq_parser.inconsistent_answer",
+                event_info="Duplicate question with different answer found.",
+                question=question,
+                previous_answers=seen_questions[question],
+                new_answer=answer,
+            )
+
+        seen_qa_pairs.add((question, answer))
+        seen_questions[question].append(answer)

rasa/core/jobs.py

@@ -1,12 +1,13 @@
 import asyncio
 import logging
+from typing import Optional
 
 from apscheduler.schedulers.asyncio import AsyncIOScheduler
 from pytz import UnknownTimeZoneError, utc
 
 import rasa.shared.utils.io
 
-__scheduler = None
+__scheduler: Optional[AsyncIOScheduler] = None
 
 logger = logging.getLogger(__name__)