rasa-pro 3.12.21__py3-none-any.whl → 3.13.0__py3-none-any.whl

rasa/core/processor.py

@@ -12,7 +12,7 @@ from typing import TYPE_CHECKING, Any, Dict, List, Optional, Text, Tuple, Union
 import structlog
 
 import rasa.core.actions.action
-import rasa.core.tracker_store
+import rasa.core.tracker_stores.tracker_store
 import rasa.core.utils
 import rasa.shared.core.trackers
 import rasa.shared.utils.io
@@ -75,8 +75,8 @@ from rasa.shared.core.constants import (
     ACTION_SESSION_START_NAME,
     FOLLOWUP_ACTION,
     SESSION_START_METADATA_SLOT,
+    SILENCE_TIMEOUT_SLOT,
     SLOT_CONSECUTIVE_SILENCE_TIMEOUTS,
-    SLOT_SILENCE_TIMEOUT,
     USER_INTENT_RESTART,
     USER_INTENT_SILENCE_TIMEOUT,
     SetSlotExtractor,
@@ -112,7 +112,8 @@ from rasa.utils.common import TempDirectoryPath, get_temp_dir_name
 from rasa.utils.endpoints import EndpointConfig
 
 if TYPE_CHECKING:
-    from rasa.core.utils import AvailableEndpoints
+    from rasa.core.available_endpoints import AvailableEndpoints
+    from rasa.privacy.privacy_manager import BackgroundPrivacyManager
 
 logger = logging.getLogger(__name__)
 structlogger = structlog.get_logger()
@@ -129,7 +130,7 @@ class MessageProcessor:
     def __init__(
         self,
         model_path: Union[Text, Path],
-        tracker_store: rasa.core.tracker_store.TrackerStore,
+        tracker_store: rasa.core.tracker_stores.tracker_store.TrackerStore,
         lock_store: LockStore,
         generator: NaturalLanguageGenerator,
         action_endpoint: Optional[EndpointConfig] = None,
@@ -138,6 +139,7 @@ class MessageProcessor:
         on_circuit_break: Optional[LambdaType] = None,
         http_interpreter: Optional[RasaNLUHttpInterpreter] = None,
         endpoints: Optional["AvailableEndpoints"] = None,
+        privacy_manager: Optional["BackgroundPrivacyManager"] = None,
     ) -> None:
         """Initializes a `MessageProcessor`."""
         self.nlg = generator
@@ -167,6 +169,9 @@ class MessageProcessor:
         self.model_path = Path(model_path)
         self.domain = self.model_metadata.domain
         self.http_interpreter = http_interpreter
+        self.privacy_manager = privacy_manager
+        if self.privacy_manager is not None:
+            self.privacy_manager.validate_sensitive_slots_in_domain(self.domain)
 
     @staticmethod
     def _load_model(
@@ -216,15 +221,36 @@ class MessageProcessor:
 
         await self._run_prediction_loop(message.output_channel, tracker)
 
-        await self.run_anonymization_pipeline(tracker)
-
         await self.save_tracker(tracker)
 
+        self.trigger_anonymization(tracker)
+
         if isinstance(message.output_channel, CollectingOutputChannel):
             return message.output_channel.messages
 
         return None
 
+    def trigger_anonymization(self, tracker: DialogueStateTracker) -> None:
+        if self.privacy_manager is None:
+            structlogger.debug(
+                "processor.trigger_anonymization.skipping.pii_management_not_enabled",
+            )
+            return None
+
+        if not self.privacy_manager.event_brokers:
+            structlogger.debug(
+                "processor.trigger_anonymization.skipping.no_event_brokers",
+            )
+            return None
+
+        structlogger.info(
+            "rasa.core.processor.trigger_anonymization",
+            sender_id=tracker.sender_id,
+            event_info="Triggering anonymization for publishing anonymized "
+            "events to the event broker.",
+        )
+        return self.privacy_manager.run(tracker)
+
     async def run_action_extract_slots(
         self,
         output_channel: OutputChannel,
@@ -265,26 +291,6 @@ class MessageProcessor:
 
         return tracker
 
-    async def run_anonymization_pipeline(self, tracker: DialogueStateTracker) -> None:
-        """Run the anonymization pipeline on the new tracker events.
-
-        Args:
-            tracker: A tracker representing a conversation state.
-        """
-        anonymization_pipeline = plugin_manager().hook.get_anonymization_pipeline()
-        if anonymization_pipeline is None:
-            return None
-
-        old_tracker = await self.tracker_store.retrieve(tracker.sender_id)
-        new_events = rasa.shared.core.trackers.TrackerEventDiffEngine.event_difference(
-            old_tracker, tracker
-        )
-
-        for event in new_events:
-            body = {"sender_id": tracker.sender_id}
-            body.update(event.as_dict())
-            anonymization_pipeline.run(body)
-
     async def predict_next_for_sender_id(
         self, sender_id: Text
     ) -> Optional[Dict[Text, Any]]:
@@ -822,28 +828,8 @@ class MessageProcessor:
         )
 
         self._check_for_unseen_features(parse_data)
-        # resetting timeouts variables whenever something that is not a timeout occurs
-        if (
-            parse_data.get(INTENT, {}).get(INTENT_NAME_KEY)
-            != USER_INTENT_SILENCE_TIMEOUT
-            and tracker
-        ):
-            if (
-                SLOT_CONSECUTIVE_SILENCE_TIMEOUTS in tracker.slots
-                and tracker.slots[SLOT_CONSECUTIVE_SILENCE_TIMEOUTS].value != 0.0
-            ):
-                tracker.update(SlotSet(SLOT_CONSECUTIVE_SILENCE_TIMEOUTS, 0.0))
-            if (
-                SLOT_SILENCE_TIMEOUT in tracker.slots
-                and tracker.slots[SLOT_SILENCE_TIMEOUT].value
-                != tracker.slots[SLOT_SILENCE_TIMEOUT].initial_value
-            ):
-                tracker.update(
-                    SlotSet(
-                        SLOT_SILENCE_TIMEOUT,
-                        tracker.slots[SLOT_SILENCE_TIMEOUT].initial_value,
-                    )
-                )
+
+        self._initialise_consecutive_silence_timeout_slots(parse_data, tracker)
 
         return parse_data
 
@@ -1068,7 +1054,14 @@ class MessageProcessor:
 
     @staticmethod
     def _should_handle_message(tracker: DialogueStateTracker) -> bool:
-        return not tracker.is_paused() or (
+        return not tracker.is_paused() or MessageProcessor._last_user_intent_is_restart(
+            tracker
+        )
+
+    @staticmethod
+    def _last_user_intent_is_restart(tracker: DialogueStateTracker) -> bool:
+        """Check if the last user intent is a restart intent."""
+        return (
             tracker.latest_message is not None
             and tracker.latest_message.intent.get(INTENT_NAME_KEY)
             == USER_INTENT_RESTART
@@ -1615,3 +1608,31 @@ class MessageProcessor:
         )
 
         return tracker, validate_frames
+
+    @staticmethod
+    def _initialise_consecutive_silence_timeout_slots(
+        parse_data: Dict[str, Any],
+        tracker: DialogueStateTracker,
+    ) -> None:
+        # resetting timeouts variables whenever something that is not a timeout occurs
+        if (
+            parse_data.get(INTENT, {}).get(INTENT_NAME_KEY)
+            != USER_INTENT_SILENCE_TIMEOUT
+            and tracker
+        ):
+            if (
+                SLOT_CONSECUTIVE_SILENCE_TIMEOUTS in tracker.slots
+                and tracker.slots[SLOT_CONSECUTIVE_SILENCE_TIMEOUTS].value != 0.0
+            ):
+                tracker.update(SlotSet(SLOT_CONSECUTIVE_SILENCE_TIMEOUTS, 0.0))
+            if (
+                SILENCE_TIMEOUT_SLOT in tracker.slots
+                and tracker.slots[SILENCE_TIMEOUT_SLOT].value
+                != tracker.slots[SILENCE_TIMEOUT_SLOT].initial_value
+            ):
+                tracker.update(
+                    SlotSet(
+                        SILENCE_TIMEOUT_SLOT,
+                        tracker.slots[SILENCE_TIMEOUT_SLOT].initial_value,
+                    )
+                )

rasa/core/run.py

@@ -30,12 +30,11 @@ from rasa import server, telemetry
 from rasa.constants import ENV_SANIC_BACKLOG
 from rasa.core import agent, channels, constants
 from rasa.core.agent import Agent
+from rasa.core.available_endpoints import AvailableEndpoints
 from rasa.core.channels import console
 from rasa.core.channels.channel import InputChannel
 from rasa.core.channels.development_inspector import DevelopmentInspectProxy
 from rasa.core.persistor import StorageType
-from rasa.core.utils import AvailableEndpoints
-from rasa.plugin import plugin_manager
 from rasa.shared.exceptions import RasaException
 from rasa.shared.utils.yaml import read_config_file
 from rasa.utils import licensing
@@ -45,7 +44,7 @@ logger = logging.getLogger()  # get the root logger
 
 def create_http_input_channels(
     channel: Optional[Text], credentials_file: Optional[Text]
-) -> List["InputChannel"]:
+) -> List[InputChannel]:
     """Instantiate the chosen input channel."""
     if credentials_file:
         all_credentials = read_config_file(credentials_file)
@@ -59,22 +58,45 @@ def create_http_input_channels(
                 "To connect to all given channels, omit the '--connector' "
                 "argument.".format(channel)
             )
-        return [_create_single_channel(channel, all_credentials.get(channel))]
+        return [
+            _create_single_channel(
+                channel,
+                all_credentials.get(channel),
+            )
+        ]
     else:
         return [_create_single_channel(c, k) for c, k in all_credentials.items()]
 
 
-def _create_single_channel(channel: Text, credentials: Dict[Text, Any]) -> Any:
+def _create_single_channel(
+    channel: Text,
+    credentials: Optional[Dict[Text, Any]],
+) -> Any:
+    """Create a single input channel based on the channel name and credentials.
+
+    Args:
+        channel: The name of the input channel to create.
+        credentials: The credentials for the input channel.
+
+    Returns:
+        An instance of the input channel class.
+
+    Raises:
+        RasaException: If the channel class cannot be found or instantiated.
+    """
     from rasa.core.channels import BUILTIN_CHANNELS
 
     if channel in BUILTIN_CHANNELS:
-        return BUILTIN_CHANNELS[channel].from_credentials(credentials)
+        channel_class = BUILTIN_CHANNELS[channel]
+
+        return channel_class.from_credentials(credentials)
     else:
         # try to load channel based on class name
         try:
             input_channel_class = rasa.shared.utils.common.class_from_module_path(
                 channel
             )
+
             return input_channel_class.from_credentials(credentials)
         except (AttributeError, ImportError):
             raise RasaException(
@@ -108,7 +130,7 @@ def _is_apple_silicon_system() -> bool:
 
 
 def configure_app(
-    input_channels: Optional[List["InputChannel"]] = None,
+    input_channels: Optional[List[InputChannel]] = None,
     cors: Optional[Union[Text, List[Text], None]] = None,
     auth_token: Optional[Text] = None,
     enable_api: bool = True,
@@ -190,10 +212,6 @@ def configure_app(
             logger.info("Killing Sanic server now.")
             running_app.stop()  # kill the sanic server
 
-    @app.after_server_stop
-    async def after_server_stop(running_app: Sanic) -> None:
-        plugin_manager().hook.after_server_stop()
-
     if server_listeners:
         for listener, event in server_listeners:
             app.register_listener(listener, event)
@@ -346,3 +364,7 @@ async def close_resources(app: Sanic, _: AbstractEventLoop) -> None:
     event_broker = current_agent.tracker_store.event_broker
     if event_broker:
         await event_broker.close()
+
+    privacy_manager = current_agent.privacy_manager
+    if privacy_manager:
+        privacy_manager.stop()

rasa/core/{auth_retry_tracker_store.py → tracker_stores/auth_retry_tracker_store.py}

@@ -3,7 +3,7 @@ from typing import Iterable, Optional, Text
 
 from rasa.core.brokers.broker import EventBroker
 from rasa.core.secrets_manager.secret_manager import EndpointResolver
-from rasa.core.tracker_store import TrackerStore, create_tracker_store
+from rasa.core.tracker_stores.tracker_store import TrackerStore, create_tracker_store
 from rasa.shared.core.domain import Domain
 from rasa.shared.core.trackers import DialogueStateTracker
 from rasa.utils.endpoints import EndpointConfig
@@ -92,6 +92,29 @@ class AuthRetryTrackerStore(TrackerStore):
             )
             return None
 
+    async def retrieve_full_tracker(
+        self, sender_id: Text
+    ) -> Optional["DialogueStateTracker"]:
+        """Retries retrieving the full tracker if it fails."""
+        # add + 1 to retries because the retries are additional to the first attempt
+        for _ in range(self.retries + 1):
+            try:
+                return await self._tracker_store.retrieve_full_tracker(sender_id)
+            except Exception as e:
+                logger.warning(
+                    f"Failed to retrieve full tracker for {sender_id}. Retrying...",
+                    exc_info=e,
+                )
+                self._tracker_store = self.recreate_tracker_store(
+                    self.domain, self.event_broker
+                )
+
+        logger.error(
+            f"Failed to retrieve full tracker for {sender_id} "
+            f"after {self.retries} retries."
+        )
+        return None
+
     async def save(self, tracker: "DialogueStateTracker") -> None:
         """Retries saving the tracker if it fails."""
         # add + 1 to retries because the retries are additional to the first attempt
@@ -119,3 +142,45 @@ class AuthRetryTrackerStore(TrackerStore):
         """Recreate tracker store with updated credentials."""
         endpoint_config = EndpointResolver.update_config(self.endpoint_config)
         return create_tracker_store(endpoint_config, domain, event_broker)
+
+    async def delete(self, sender_id: str) -> None:
+        """Retries deleting the tracker for the given sender_id."""
+        # add + 1 to retries because the retries are additional to the first attempt
+        for _ in range(self.retries + 1):
+            try:
+                await self._tracker_store.delete(sender_id)
+                break
+            except Exception as e:
+                logger.warning(
+                    f"Failed to delete tracker for {sender_id}. Retrying...",
+                    exc_info=e,
+                )
+                self._tracker_store = self.recreate_tracker_store(
+                    self.domain, self.event_broker
+                )
+        else:
+            logger.error(
+                f"Failed to delete tracker for {sender_id} "
+                f"after {self.retries} retries."
+            )
+
+    async def update(self, tracker: DialogueStateTracker) -> None:
+        """Retries replacing the tracker if it fails."""
+        # add + 1 to retries because the retries are additional to the first attempt
+        for _ in range(self.retries + 1):
+            try:
+                await self._tracker_store.update(tracker)
+                break
+            except Exception as e:
+                logger.warning(
+                    f"Failed to replace tracker for {tracker.sender_id}. Retrying...",
+                    exc_info=e,
+                )
+                self._tracker_store = self.recreate_tracker_store(
+                    self.domain, self.event_broker
+                )
+        else:
+            logger.error(
+                f"Failed to replace tracker for {tracker.sender_id} "
+                f"after {self.retries} retries."
+            )

rasa/core/tracker_stores/dynamo_tracker_store.py

@@ -0,0 +1,256 @@
+from __future__ import annotations
+
+import os
+from typing import TYPE_CHECKING, Any, Dict, Iterable, Optional, Text
+
+import structlog
+from boto3.dynamodb.conditions import Key
+
+import rasa.utils
+from rasa.constants import DEFAULT_SANIC_WORKERS, ENV_SANIC_WORKERS
+from rasa.core.tracker_stores.tracker_store import (
+    SerializedTrackerAsDict,
+    TrackerStore,
+)
+from rasa.shared.core.domain import Domain
+from rasa.shared.core.trackers import DialogueStateTracker
+from rasa.shared.exceptions import RasaException
+from rasa.utils.endpoints import EndpointConfig
+
+structlogger = structlog.get_logger(__name__)
+
+if TYPE_CHECKING:
+    import boto3.resources.factory.dynamodb.Table
+
+
+class DynamoTrackerStore(TrackerStore, SerializedTrackerAsDict):
+    """Stores conversation history in DynamoDB."""
+
+    def __init__(
+        self,
+        domain: Domain,
+        table_name: Text = "states",
+        region: Text = "us-east-1",
+        event_broker: Optional[EndpointConfig] = None,
+        **kwargs: Dict[Text, Any],
+    ) -> None:
+        """Initialize `DynamoTrackerStore`.
+
+        Args:
+            domain: Domain associated with this tracker store.
+            table_name: The name of the DynamoDB table, does not need to be present a
+                priori.
+            region: The name of the region associated with the client.
+                A client is associated with a single region.
+            event_broker: An event broker used to publish events.
+            kwargs: Additional kwargs.
+        """
+        import boto3
+
+        self.client = boto3.client("dynamodb", region_name=region)
+        self.region = region
+        self.table_name = table_name
+        self.db = self.get_or_create_table(table_name)
+        super().__init__(domain, event_broker, **kwargs)
+
+    def get_or_create_table(
+        self, table_name: Text
+    ) -> "boto3.resources.factory.dynamodb.Table":
+        """Returns table or creates one if the table name is not in the table list."""
+        import boto3
+
+        dynamo = boto3.resource("dynamodb", region_name=self.region)
+        try:
+            self.client.describe_table(TableName=table_name)
+        except self.client.exceptions.ResourceNotFoundException:
+            sanic_workers_count = int(
+                os.environ.get(ENV_SANIC_WORKERS, DEFAULT_SANIC_WORKERS)
+            )
+
+            if sanic_workers_count > 1:
+                structlogger.error(
+                    "dynamo_tracker_store.table_creation_not_supported_in_multi_worker_mode",
+                    event_info=(
+                        "DynamoDB table creation is not "
+                        "supported in multi-worker mode. "
+                        "Table should already exist.",
+                    ),
+                )
+                raise RasaException(
+                    "DynamoDB table creation is not supported in "
+                    "case of multiple sanic workers. To create the table either "
+                    "run Rasa with a single worker or create the table manually."
+                    "Here are the defaults which can be used to "
+                    "create the table manually: "
+                    f"Table name: {table_name}, Primary key: sender_id, "
+                    f"key type `HASH`, attribute type `S` (String), "
+                    "Provisioned throughput: Read capacity units: 5, "
+                    "Write capacity units: 5"
+                )
+
+            table = dynamo.create_table(
+                TableName=self.table_name,
+                KeySchema=[{"AttributeName": "sender_id", "KeyType": "HASH"}],
+                AttributeDefinitions=[
+                    {"AttributeName": "sender_id", "AttributeType": "S"}
+                ],
+                ProvisionedThroughput={"ReadCapacityUnits": 5, "WriteCapacityUnits": 5},
+            )
+
+            # Wait until the table exists.
+            table.meta.client.get_waiter("table_exists").wait(TableName=table_name)
+        else:
+            table = dynamo.Table(table_name)
+
+        return table
+
+    async def save(self, tracker: DialogueStateTracker) -> None:
+        """Saves the current conversation state."""
+        await self.stream_events(tracker)
+        serialized = self.serialise_tracker(tracker)
+
+        full_tracker = await self.retrieve_full_tracker(tracker.sender_id)
+        if full_tracker is None:
+            self.db.put_item(Item=serialized)
+            return None
+
+        # return the latest events since the last user message
+        new_tracker = DialogueStateTracker.from_dict(
+            serialized["sender_id"], events_as_dict=serialized["events"]
+        )
+        new_events = new_tracker.get_last_turn_events()
+        new_serialized_events = [event.as_dict() for event in new_events]
+
+        # we need to save the full tracker if it is a new tracker
+        # without events following a user message
+        if not new_serialized_events:
+            self.db.put_item(Item=serialized)
+            return None
+
+        # append new events to the existing tracker
+        self.db.update_item(
+            Key={"sender_id": tracker.sender_id},
+            UpdateExpression="SET events = list_append(if_not_exists(events, :empty_list), :events)",  # noqa: E501
+            ExpressionAttributeValues={
+                ":events": new_serialized_events,
+                ":empty_list": [],
+            },
+            ReturnValues="UPDATED_NEW",
+        )
+        return None
+
+    async def delete(self, sender_id: Text) -> None:
+        """Delete tracker for the given sender_id."""
+        if not await self.exists(sender_id):
+            structlogger.info(
+                "dynamo_tracker_store.delete.no_tracker_for_sender_id",
+                event_info=f"Could not find tracker for conversation ID '{sender_id}'.",
+            )
+            return None
+
+        self.db.delete_item(
+            Key={"sender_id": sender_id},
+            ConditionExpression="attribute_exists(sender_id)",
+        )
+        structlogger.info(
+            "dynamo_tracker_store.delete.deleted_tracker",
+            sender_id=sender_id,
+        )
+
+    @staticmethod
+    def serialise_tracker(
+        tracker: "DialogueStateTracker",
+    ) -> Dict:
+        """Serializes the tracker, returns object with decimal types.
+
+        DynamoDB cannot store `float`s, so we'll convert them to `Decimal`s.
+        """
+        return rasa.utils.json_utils.replace_floats_with_decimals(
+            SerializedTrackerAsDict.serialise_tracker(tracker)
+        )
+
+    async def retrieve(self, sender_id: Text) -> Optional[DialogueStateTracker]:
+        """Retrieve dialogues for a sender_id in reverse-chronological order.
+
+        Based on the session_date sort key.
+        """
+        return await self._retrieve(sender_id, fetch_all_sessions=False)
+
+    async def retrieve_full_tracker(
+        self, sender_id: Text
+    ) -> Optional[DialogueStateTracker]:
+        """Retrieves tracker for all conversation sessions.
+
+        Args:
+            sender_id: Conversation ID to fetch the tracker for.
+        """
+        return await self._retrieve(sender_id, fetch_all_sessions=True)
+
+    async def _retrieve(
+        self, sender_id: Text, fetch_all_sessions: bool
+    ) -> Optional[DialogueStateTracker]:
+        """Returns tracker matching sender_id.
+
+        Args:
+            sender_id: Conversation ID to fetch the tracker for.
+            fetch_all_sessions: Whether to fetch all sessions or only the last one.
+        """
+        dialogues = self.db.query(
+            KeyConditionExpression=Key("sender_id").eq(sender_id),
+            ScanIndexForward=False,
+        )["Items"]
+
+        if not dialogues:
+            return None
+
+        events_with_floats = []
+        for dialogue in dialogues:
+            if dialogue.get("events"):
+                events = rasa.utils.json_utils.replace_decimals_with_floats(
+                    dialogue["events"]
+                )
+                events_with_floats.extend(events)
+
+        if self.domain is None:
+            slots = []
+        else:
+            slots = self.domain.slots
+
+        tracker = DialogueStateTracker.from_dict(sender_id, events_with_floats, slots)
+
+        if fetch_all_sessions:
+            return tracker
+
+        # only return the last session
+        multiple_tracker_sessions = (
+            rasa.shared.core.trackers.get_trackers_for_conversation_sessions(tracker)
+        )
+
+        if len(multiple_tracker_sessions) <= 1:
+            return tracker
+
+        return multiple_tracker_sessions[-1]
+
+    async def keys(self) -> Iterable[Text]:
+        """Returns sender_ids of the `DynamoTrackerStore`."""
+        response = self.db.scan(ProjectionExpression="sender_id")
+        sender_ids = [i["sender_id"] for i in response["Items"]]
+
+        while response.get("LastEvaluatedKey"):
+            response = self.db.scan(
+                ProjectionExpression="sender_id",
+                ExclusiveStartKey=response["LastEvaluatedKey"],
+            )
+            sender_ids.extend([i["sender_id"] for i in response["Items"]])
+
+        return sender_ids
+
+    async def update(self, tracker: DialogueStateTracker) -> None:
+        """Overwrites the tracker for the given sender_id."""
+        serialized = self.serialise_tracker(tracker)
+        self.db.put_item(Item=serialized)
+
+        structlogger.info(
+            "dynamo_tracker_store.replace.replaced_tracker",
+            sender_id=tracker.sender_id,
+        )