rasa-pro 3.13.0rc3__py3-none-any.whl → 3.13.1a2__py3-none-any.whl

rasa/core/channels/studio_chat.py

@@ -49,7 +49,7 @@ if TYPE_CHECKING:
 structlogger = structlog.get_logger()
 
 
-def tracker_as_dump(tracker: "DialogueStateTracker") -> str:
+def tracker_as_dump(tracker: "DialogueStateTracker") -> Dict[str, Any]:
     """Create a dump of the tracker state."""
     from rasa.shared.core.trackers import get_trackers_for_conversation_sessions
 
@@ -60,8 +60,9 @@ def tracker_as_dump(tracker: "DialogueStateTracker") -> str:
     else:
         last_tracker = multiple_tracker_sessions[-1]
 
-    state = last_tracker.current_state(EventVerbosity.AFTER_RESTART)
-    return json.dumps(state)
+    # TODO: this is a bug: the bridge converts this back to json, but it
+    # should be json in the first place
+    return last_tracker.current_state(EventVerbosity.AFTER_RESTART)
 
 
 def does_need_action_prediction(tracker: "DialogueStateTracker") -> bool:
@@ -148,6 +149,7 @@ class StudioChatInput(SocketIOInput, VoiceInputChannel):
         from rasa.core.agent import Agent
 
         self.agent: Optional[Agent] = None
+        self.latest_tracker_session_id = None
 
         # Initialize the SocketIO input channel
         SocketIOInput.__init__(
@@ -211,6 +213,11 @@ class StudioChatInput(SocketIOInput, VoiceInputChannel):
         if not self.sio:
             structlogger.error("studio_chat.on_tracker_updated.sio_not_initialized")
             return
+
+        # we need the latest session id to use it for the llm helper to get the
+        # most recent conversation the user had with the bot.
+        self.latest_tracker_session_id = sender_id
+
         await self.sio.emit("tracker", tracker_dump, room=sender_id)
 
     async def on_message_proxy(
@@ -222,7 +229,14 @@ class StudioChatInput(SocketIOInput, VoiceInputChannel):
 
         Triggers a tracker update notification after processing the message.
         """
-        await on_new_message(message)
+        try:
+            await on_new_message(message)
+        except Exception as e:
+            structlogger.exception(
+                "studio_chat.on_new_message.error",
+                error=str(e),
+                sender_id=message.sender_id,
+            )
 
         if not self.agent:
             structlogger.error("studio_chat.on_message_proxy.agent_not_initialized")
@@ -275,7 +289,7 @@ class StudioChatInput(SocketIOInput, VoiceInputChannel):
     async def collect_call_parameters(
         self, channel_websocket: "Websocket"
     ) -> Optional[CallParameters]:
-        """Voice method to collect call parameters"""
+        """Voice method to collect call parameters."""
         session_id = channel_websocket.session_id
         return CallParameters(session_id, "local", "local", stream_id=session_id)
 
@@ -305,7 +319,7 @@ class StudioChatInput(SocketIOInput, VoiceInputChannel):
     def create_output_channel(
         self, voice_websocket: "Websocket", tts_engine: TTSEngine
     ) -> VoiceOutputChannel:
-        """Create a voice output channel"""
+        """Create a voice output channel."""
         return StudioVoiceOutputChannel(
             voice_websocket,
             tts_engine,
@@ -432,8 +446,15 @@ class StudioChatInput(SocketIOInput, VoiceInputChannel):
                     ws.put_message(data)
                 return
 
-            # Handle text messages
-            await self.handle_user_message(sid, data, on_new_message)
+            try:
+                # Handle text messages
+                await self.handle_user_message(sid, data, on_new_message)
+            except Exception as e:
+                structlogger.exception(
+                    "studio_chat.sio.handle_message.error",
+                    error=str(e),
+                    sid=sid,
+                )
 
         @self.sio.on("update_tracker", namespace=self.namespace)
         async def on_update_tracker(sid: Text, data: Dict) -> None:

rasa/core/policies/flows/flow_executor.py

@@ -740,7 +740,14 @@ def _run_action_step(
             # do not log about non-existing validation actions of collect steps
             utter_action_name = render_template_variables("{{context.utter}}", context)
             if utter_action_name not in available_actions:
-                structlogger.warning("flow.step.run.action.unknown", action=action_name)
+                structlogger.warning(
+                    "flow.step.run.action.unknown",
+                    action=action_name,
+                    event_info=(
+                        f"The action '{action_name}' is not defined in the domain but "
+                        f"getting triggered by the flow '{step.flow_id}'."
+                    ),
+                )
         return ContinueFlowWithNextStep(events=initial_events)
 
 

rasa/core/tracker_stores/auth_retry_tracker_store.py

@@ -92,6 +92,29 @@ class AuthRetryTrackerStore(TrackerStore):
             )
             return None
 
+    async def retrieve_full_tracker(
+        self, sender_id: Text
+    ) -> Optional["DialogueStateTracker"]:
+        """Retries retrieving the full tracker if it fails."""
+        # add + 1 to retries because the retries are additional to the first attempt
+        for _ in range(self.retries + 1):
+            try:
+                return await self._tracker_store.retrieve_full_tracker(sender_id)
+            except Exception as e:
+                logger.warning(
+                    f"Failed to retrieve full tracker for {sender_id}. Retrying...",
+                    exc_info=e,
+                )
+                self._tracker_store = self.recreate_tracker_store(
+                    self.domain, self.event_broker
+                )
+
+        logger.error(
+            f"Failed to retrieve full tracker for {sender_id} "
+            f"after {self.retries} retries."
+        )
+        return None
+
     async def save(self, tracker: "DialogueStateTracker") -> None:
         """Retries saving the tracker if it fails."""
         # add + 1 to retries because the retries are additional to the first attempt
@@ -120,6 +143,44 @@ class AuthRetryTrackerStore(TrackerStore):
         endpoint_config = EndpointResolver.update_config(self.endpoint_config)
         return create_tracker_store(endpoint_config, domain, event_broker)
 
-    async def delete(self, sender_id: Text) -> None:
-        """Delete tracker for the given sender_id."""
-        await self._tracker_store.delete(sender_id)
+    async def delete(self, sender_id: str) -> None:
+        """Retries deleting the tracker for the given sender_id."""
+        # add + 1 to retries because the retries are additional to the first attempt
+        for _ in range(self.retries + 1):
+            try:
+                await self._tracker_store.delete(sender_id)
+                break
+            except Exception as e:
+                logger.warning(
+                    f"Failed to delete tracker for {sender_id}. Retrying...",
+                    exc_info=e,
+                )
+                self._tracker_store = self.recreate_tracker_store(
+                    self.domain, self.event_broker
+                )
+        else:
+            logger.error(
+                f"Failed to delete tracker for {sender_id} "
+                f"after {self.retries} retries."
+            )
+
+    async def update(self, tracker: DialogueStateTracker) -> None:
+        """Retries replacing the tracker if it fails."""
+        # add + 1 to retries because the retries are additional to the first attempt
+        for _ in range(self.retries + 1):
+            try:
+                await self._tracker_store.update(tracker)
+                break
+            except Exception as e:
+                logger.warning(
+                    f"Failed to replace tracker for {tracker.sender_id}. Retrying...",
+                    exc_info=e,
+                )
+                self._tracker_store = self.recreate_tracker_store(
+                    self.domain, self.event_broker
+                )
+        else:
+            logger.error(
+                f"Failed to replace tracker for {tracker.sender_id} "
+                f"after {self.retries} retries."
+            )

rasa/core/tracker_stores/dynamo_tracker_store.py

@@ -216,3 +216,13 @@ class DynamoTrackerStore(TrackerStore, SerializedTrackerAsDict):
             sender_ids.extend([i["sender_id"] for i in response["Items"]])
 
         return sender_ids
+
+    async def update(self, tracker: DialogueStateTracker) -> None:
+        """Overwrites the tracker for the given sender_id."""
+        serialized = self.serialise_tracker(tracker)
+        self.db.put_item(Item=serialized)
+
+        structlogger.info(
+            "dynamo_tracker_store.replace.replaced_tracker",
+            sender_id=tracker.sender_id,
+        )

rasa/core/tracker_stores/mongo_tracker_store.py

@@ -1,6 +1,7 @@
 from __future__ import annotations
 
 import itertools
+from datetime import datetime
 from typing import Any, Dict, Iterable, Iterator, List, Optional, Text
 
 import structlog
@@ -204,3 +205,19 @@ class MongoTrackerStore(TrackerStore, SerializedTrackerAsText):
     async def keys(self) -> Iterable[Text]:
         """Returns sender_ids of the Mongo Tracker Store."""
         return [c["sender_id"] for c in self.conversations.find()]
+
+    async def update(self, tracker: DialogueStateTracker) -> None:
+        """Overwrites the tracker for the given sender_id."""
+        self.conversations.replace_one(
+            {"sender_id": tracker.sender_id},
+            tracker.current_state(EventVerbosity.ALL),
+            upsert=True,
+        )
+
+        first_event_timestamp = str(datetime.fromtimestamp(tracker.events[0].timestamp))
+
+        structlogger.info(
+            "redis_tracker_store.update.updated_tracker",
+            sender_id=tracker.sender_id,
+            first_event_timestamp=first_event_timestamp,
+        )

rasa/core/tracker_stores/redis_tracker_store.py

@@ -1,5 +1,6 @@
 from __future__ import annotations
 
+from datetime import datetime
 from typing import Any, Dict, Iterable, Optional, Text
 
 import structlog
@@ -227,3 +228,25 @@ class RedisTrackerStore(TrackerStore, SerializedTrackerAsText):
                 merged.update(new_event)
 
         return merged
+
+    async def update(self, tracker: DialogueStateTracker) -> None:
+        """Overwrites the tracker for the given sender_id."""
+        serialised_tracker = self.serialise_tracker(tracker)
+
+        # if the sender_id starts with the key prefix, we remove it
+        # this is used to avoid storing the prefix twice
+        sender_id = tracker.sender_id
+        if sender_id.startswith(self.key_prefix):
+            sender_id = sender_id[len(self.key_prefix) :]
+
+        self.red.set(
+            self.key_prefix + sender_id, serialised_tracker, ex=self.record_exp
+        )
+
+        first_event_timestamp = str(datetime.fromtimestamp(tracker.events[0].timestamp))
+
+        structlogger.info(
+            "redis_tracker_store.update.updated_tracker",
+            sender_id=tracker.sender_id,
+            first_event_timestamp=first_event_timestamp,
+        )

rasa/core/tracker_stores/sql_tracker_store.py

@@ -4,6 +4,7 @@ import contextlib
 import itertools
 import json
 import os
+from datetime import datetime
 from time import sleep
 from typing import (
     TYPE_CHECKING,
@@ -553,3 +554,29 @@ class SQLTrackerStore(TrackerStore, SerializedTrackerAsText):
         return itertools.islice(
             tracker.events, number_of_events_since_last_session, len(tracker.events)
         )
+
+    async def update(self, tracker_to_keep: DialogueStateTracker) -> None:
+        """Overwrite the tracker in the SQL tracker store."""
+        with self.session_scope() as session:
+            # Delete events whose timestamp are older
+            # than the first event of the tracker to keep.
+            statement = sa.delete(self.SQLEvent).where(
+                self.SQLEvent.sender_id == tracker_to_keep.sender_id,
+                self.SQLEvent.timestamp < tracker_to_keep.events[0].timestamp
+                if tracker_to_keep.events
+                else 0,
+            )
+
+            result = session.execute(statement)
+            session.commit()
+
+        first_event_timestamp = str(
+            datetime.fromtimestamp(tracker_to_keep.events[0].timestamp)
+        )
+
+        structlogger.info(
+            "sql_tracker_store.update.updated_tracker",
+            sender_id=tracker_to_keep.sender_id,
+            first_event_timestamp=first_event_timestamp,
+            event_info=f"{result.rowcount} rows removed from tracker.",
+        )

rasa/core/tracker_stores/tracker_store.py

@@ -251,6 +251,14 @@ class TrackerStore:
         """
         raise NotImplementedError()
 
+    async def update(self, tracker: DialogueStateTracker) -> None:
+        """Replace an existing tracker with a new one.
+
+        Args:
+            tracker: The tracker to update.
+        """
+        raise NotImplementedError()
+
     async def retrieve_full_tracker(
         self, conversation_id: Text
     ) -> Optional[DialogueStateTracker]:
@@ -478,6 +486,14 @@ class InMemoryTrackerStore(TrackerStore, SerializedTrackerAsText):
 
         return multiple_tracker_sessions[-1]
 
+    async def update(self, tracker: DialogueStateTracker) -> None:
+        """Replace an existing tracker with a new one.
+
+        Args:
+            tracker: The tracker to update.
+        """
+        await self.save(tracker)
+
 
 def validate_port(port: Any) -> Optional[int]:
     """Ensure that port can be converted to integer.
@@ -583,7 +599,19 @@ class FailSafeTrackerStore(TrackerStore):
 
     async def delete(self, sender_id: Text) -> None:
         """Delete tracker for the given sender_id."""
-        await self._tracker_store.delete(sender_id)
+        try:
+            await self._tracker_store.delete(sender_id)
+        except Exception as e:
+            self.on_tracker_store_error(e)
+            await self.fallback_tracker_store.delete(sender_id)
+
+    async def update(self, tracker: DialogueStateTracker) -> None:
+        """Replace an existing tracker with a new one."""
+        try:
+            await self._tracker_store.update(tracker)
+        except Exception as e:
+            self.on_tracker_store_error(e)
+            await self.fallback_tracker_store.update(tracker)
 
     async def retrieve_full_tracker(
         self, sender_id: Text
@@ -793,7 +821,13 @@ class AwaitableTrackerStore(TrackerStore):
 
     async def delete(self, sender_id: Text) -> None:
         """Delete tracker for the given sender_id."""
-        await self._tracker_store.delete(sender_id)
+        result = self._tracker_store.delete(sender_id)
+        return await result if isawaitable(result) else result
+
+    async def update(self, tracker: DialogueStateTracker) -> None:
+        """Replace an existing tracker with a new one."""
+        result = self._tracker_store.update(tracker)
+        return await result if isawaitable(result) else result
 
     async def retrieve_full_tracker(
         self, conversation_id: Text

rasa/dialogue_understanding/patterns/default_flows_for_patterns.yml

@@ -249,7 +249,7 @@ flows:
                 next: END
 
   pattern_repeat_bot_messages:
-    description: Voice conversation repair pattern to repeat bot messages
+    description: Conversation repair flow for repeating previous messages
     name: pattern repeat bot messages
     steps:
       - action: action_repeat_bot_messages

rasa/model_manager/model_api.py

@@ -43,7 +43,6 @@ from rasa.model_manager.utils import (
     get_logs_content,
     logs_base_path,
     models_base_path,
-    subpath,
 )
 from rasa.model_manager.warm_rasa_process import (
     initialize_warm_rasa_process,
@@ -53,6 +52,7 @@ from rasa.server import ErrorResponse
 from rasa.shared.exceptions import InvalidConfigException
 from rasa.shared.utils.yaml import dump_obj_as_yaml_to_string
 from rasa.studio.upload import build_calm_import_parts
+from rasa.utils.io import subpath
 
 dotenv.load_dotenv()
 
@@ -105,7 +105,7 @@ async def update_status_of_all_bots() -> None:
         await update_bot_status(bot)
 
 
-def base_server_url(request: Request) -> str:
+def base_server_url(request: Optional[Request]) -> str:
     """Return the base URL of the server."""
     if SERVER_BASE_URL:
         return SERVER_BASE_URL.rstrip("/")

rasa/model_manager/runner_service.py

@@ -15,11 +15,11 @@ from rasa.model_manager import config
 from rasa.model_manager.utils import (
     logs_path,
     models_base_path,
-    subpath,
     write_encoded_data_to_file,
 )
 from rasa.model_manager.warm_rasa_process import start_rasa_process
 from rasa.studio.prompts import handle_prompts
+from rasa.utils.io import subpath
 
 structlogger = structlog.get_logger()
 

rasa/model_manager/trainer_service.py

@@ -14,7 +14,6 @@ from rasa.model_manager.utils import (
     ensure_base_directory_exists,
     logs_path,
     models_base_path,
-    subpath,
     write_encoded_data_to_file,
 )
 from rasa.model_manager.warm_rasa_process import (
@@ -22,6 +21,7 @@ from rasa.model_manager.warm_rasa_process import (
 )
 from rasa.model_training import generate_random_model_name
 from rasa.studio.prompts import handle_prompts
+from rasa.utils.io import subpath
 
 structlogger = structlog.get_logger()
 
@@ -53,6 +53,15 @@ class TrainingSession(BaseModel):
         """Check if the training is running."""
         return self.status == TrainingSessionStatus.RUNNING
 
+    def has_just_finished(self) -> bool:
+        if not self.is_status_indicating_alive():
+            # skip if the training is not running
+            return False
+        if self.process.poll() is None:
+            # process is still running
+            return False
+        return True
+
     def model_path(self) -> str:
         """Return the path to the model."""
         return subpath(models_base_path(), f"{self.model_name}.tar.gz")
@@ -89,14 +98,8 @@ def terminate_training(training: TrainingSession) -> None:
 
 
 def update_training_status(training: TrainingSession) -> None:
-    if not training.is_status_indicating_alive():
-        # skip if the training is not running
-        return
-    if training.process.poll() is None:
-        # process is still running
-        return
-
-    complete_training(training)
+    if training.has_just_finished():
+        complete_training(training)
 
 
 def complete_training(training: TrainingSession) -> None:

rasa/model_manager/utils.py

@@ -5,15 +5,11 @@ from typing import Optional
 import structlog
 
 from rasa.model_manager import config
-from rasa.shared.exceptions import RasaException
+from rasa.utils.io import subpath
 
 structlogger = structlog.get_logger()
 
 
-class InvalidPathException(RasaException):
-    """Raised if a path is invalid - e.g. path traversal is detected."""
-
-
 def write_encoded_data_to_file(encoded_data: bytes, file: str) -> None:
     """Write base64 encoded data to a file."""
     # create the directory if it does not exist of the parent directory
@@ -53,30 +49,6 @@ def logs_path(action_id: str) -> str:
     return subpath(logs_base_path(), f"{action_id}.txt")
 
 
-def subpath(parent: str, child: str) -> str:
-    """Return the path to the child directory of the parent directory.
-
-    Ensures, that child doesn't navigate to parent directories. Prevents
-    path traversal. Raises an InvalidPathException if the path is invalid.
-
-    Based on Snyk's directory traversal mitigation:
-    https://learn.snyk.io/lesson/directory-traversal/
-    """
-    safe_path = os.path.abspath(os.path.join(parent, child))
-    parent = os.path.abspath(parent)
-
-    common_base = os.path.commonpath([parent, safe_path])
-    if common_base != parent:
-        raise InvalidPathException(f"Invalid path: {safe_path}")
-
-    if os.path.basename(safe_path) != child:
-        raise InvalidPathException(
-            f"Invalid path - path traversal detected: {safe_path}"
-        )
-
-    return safe_path
-
-
 def get_logs_content(action_id: str) -> Optional[str]:
     """Return the content of the log file for a given action id."""
     try:

rasa/privacy/privacy_manager.py

@@ -12,7 +12,7 @@ import structlog
 from apscheduler.schedulers.background import BackgroundScheduler
 
 import rasa.shared.core.trackers
-from rasa.core.tracker_stores.tracker_store import FailSafeTrackerStore, TrackerStore
+from rasa.core.tracker_stores.tracker_store import TrackerStore
 from rasa.privacy.constants import (
     TEXT_KEY,
     USER_CHAT_INACTIVITY_IN_MINUTES_ENV_VAR_NAME,
@@ -25,7 +25,7 @@ from rasa.privacy.privacy_config import (
 )
 from rasa.privacy.privacy_filter import PrivacyFilter
 from rasa.shared.core.events import Event, SlotSet, UserUttered, split_events
-from rasa.shared.core.trackers import DialogueStateTracker
+from rasa.shared.core.trackers import DialogueStateTracker, EventVerbosity
 
 if TYPE_CHECKING:
     from asyncio import AbstractEventLoop
@@ -97,7 +97,7 @@ class BackgroundPrivacyManager:
                 else TrackerStore.create(None)
             )
 
-        self.tracker_store = FailSafeTrackerStore(tracker_store)
+        self.tracker_store = tracker_store
 
         self.event_brokers: List["EventBroker"] = []
         self.event_loop = event_loop
@@ -264,15 +264,16 @@ class BackgroundPrivacyManager:
             )
             return None
 
-        latest_message = tracker.latest_message
-
-        if latest_message is None or not latest_message.text:
+        latest_user_message = tracker.get_last_event_for(
+            UserUttered, event_verbosity=EventVerbosity.ALL
+        )
+        if latest_user_message is None or not latest_user_message.text:
             structlogger.debug(
                 "rasa.privacy_manager.no_user_message.skipping_processing",
             )
             return None
 
-        return latest_message
+        return latest_user_message
 
     @staticmethod
     def _has_session_been_anonymized(events: List[Event]) -> bool:
@@ -360,9 +361,13 @@ class BackgroundPrivacyManager:
                 full_tracker
             )
 
-            await self.tracker_store.delete(sender_id=key)
-
             if not events_to_be_retained:
+                await self.tracker_store.delete(sender_id=key)
+                structlogger.info(
+                    "rasa.privacy_manager.tracker_session_deleted",
+                    sender_id=full_tracker.sender_id,
+                    triggered_by="deletion_cron_job",
+                )
                 continue
 
             tracker = DialogueStateTracker.from_events(
@@ -370,12 +375,13 @@ class BackgroundPrivacyManager:
                 evts=events_to_be_retained,
                 slots=full_tracker.slots.values(),
             )
-            await self.tracker_store.save(tracker)
+            await self.tracker_store.update(tracker)
 
             structlogger.info(
-                "rasa.privacy_manager.save_tracker_after_deletion",
+                "rasa.privacy_manager.overwritten_tracker",
                 sender_id=key,
-                event_info="Saved tracker with events not scheduled "
+                event_info="Deleted eligible events and saved "
+                "tracker with events not scheduled "
                 "for deletion yet.",
             )
 
@@ -527,10 +533,7 @@ class BackgroundPrivacyManager:
                     last_event_timestamp=last_event_timestamp,
                     triggered_by="anonymization_cron_job",
                 )
-                tracker = DialogueStateTracker.from_events(
-                    session.sender_id, session.events
-                )
-                events = self.process_events(tracker, process_all=True)
+                events = self.process_events(session, process_all=True)
                 processed_events.extend(events)
             else:
                 # If the session is not valid for anonymization,