rasa-pro 3.12.0.dev1__py3-none-any.whl

rasa/core/utils.py

@@ -0,0 +1,366 @@
+import structlog
+import logging
+import os
+from pathlib import Path
+from socket import SOCK_DGRAM, SOCK_STREAM
+from typing import Any, Dict, Optional, List, Set, TYPE_CHECKING, Text, Tuple, Union
+
+import numpy as np
+from sanic import Sanic
+
+import rasa.cli.utils as cli_utils
+import rasa.shared.utils.io
+from rasa.constants import DEFAULT_SANIC_WORKERS, ENV_SANIC_WORKERS
+from rasa.core.constants import (
+    DOMAIN_GROUND_TRUTH_METADATA_KEY,
+    UTTER_SOURCE_METADATA_KEY,
+    ACTIVE_FLOW_METADATA_KEY,
+    STEP_ID_METADATA_KEY,
+)
+from rasa.core.lock_store import LockStore, RedisLockStore, InMemoryLockStore
+from rasa.shared.constants import DEFAULT_ENDPOINTS_PATH, TCP_PROTOCOL
+from rasa.shared.core.trackers import DialogueStateTracker
+from rasa.utils.endpoints import (
+    EndpointConfig,
+    read_endpoint_config,
+    read_property_config_from_endpoints_file,
+)
+from rasa.utils.io import write_yaml
+
+if TYPE_CHECKING:
+    from rasa.core.nlg import NaturalLanguageGenerator
+    from rasa.shared.core.domain import Domain
+
+structlogger = structlog.get_logger()
+
+
+def configure_file_logging(
+    logger_obj: logging.Logger,
+    log_file: Optional[Text],
+    use_syslog: Optional[bool],
+    syslog_address: Optional[Text] = None,
+    syslog_port: Optional[int] = None,
+    syslog_protocol: Optional[Text] = None,
+) -> None:
+    """Configure logging to a file.
+
+    Args:
+        logger_obj: Logger object to configure.
+        log_file: Path of log file to write to.
+        use_syslog: Add syslog as a logger.
+        syslog_address: Adress of the syslog server.
+        syslog_port: Port of the syslog server.
+        syslog_protocol: Protocol with the syslog server
+    """
+    if use_syslog:
+        formatter = logging.Formatter(
+            "%(asctime)s [%(levelname)-5.5s] [%(process)d]" " %(message)s"
+        )
+        socktype = SOCK_STREAM if syslog_protocol == TCP_PROTOCOL else SOCK_DGRAM
+        syslog_handler = logging.handlers.SysLogHandler(
+            address=(syslog_address, syslog_port), socktype=socktype
+        )
+        syslog_handler.setLevel(logger_obj.level)
+        syslog_handler.setFormatter(formatter)
+        logger_obj.addHandler(syslog_handler)
+    if log_file:
+        formatter = logging.Formatter("%(asctime)s [%(levelname)-5.5s]  %(message)s")
+        file_handler = logging.FileHandler(
+            log_file, encoding=rasa.shared.utils.io.DEFAULT_ENCODING
+        )
+        file_handler.setLevel(logger_obj.level)
+        file_handler.setFormatter(formatter)
+        logger_obj.addHandler(file_handler)
+
+
+def one_hot(hot_idx: int, length: int, dtype: Optional[Text] = None) -> np.ndarray:
+    """Create a one-hot array.
+
+    Args:
+        hot_idx: Index of the hot element.
+        length: Length of the array.
+        dtype: ``numpy.dtype`` of the array.
+
+    Returns:
+        One-hot array.
+    """
+    if hot_idx >= length:
+        raise ValueError(
+            "Can't create one hot. Index '{}' is out of range (length '{}')".format(
+                hot_idx, length
+            )
+        )
+    r = np.zeros(length, dtype)
+    r[hot_idx] = 1
+    return r
+
+
+def dump_obj_as_yaml_to_file(
+    filename: Union[Text, Path], obj: Any, should_preserve_key_order: bool = False
+) -> None:
+    """Writes `obj` to the filename in YAML repr.
+
+    Args:
+        filename: Target filename.
+        obj: Object to dump.
+        should_preserve_key_order: Whether to preserve key order in `obj`.
+    """
+    write_yaml(obj, filename, should_preserve_key_order=should_preserve_key_order)
+
+
+def list_routes(app: Sanic) -> Dict[Text, Text]:
+    """List all the routes of a sanic application. Mainly used for debugging."""
+    from urllib.parse import unquote
+
+    output = {}
+
+    def find_route(suffix: Text, path: Text) -> Optional[Text]:
+        for name, (uri, _) in app.router.routes_names.items():
+            if name.split(".")[-1] == suffix and uri == path:
+                return name
+        return None
+
+    for route in app.router.routes:
+        endpoint = route.parts
+        if endpoint[:-1] in app.router.routes_all and endpoint[-1] == "/":
+            continue
+
+        options = {}
+        for arg in route._params:
+            options[arg] = f"[{arg}]"
+
+        name = route.name.replace("rasa_server.", "")
+        methods = ",".join(route.methods)
+
+        full_endpoint = "/" + "/".join(endpoint)
+        line = unquote(f"{full_endpoint:50s} {methods:30s} {name}")
+        output[name] = line
+
+    url_table = "\n".join(output[url] for url in sorted(output))
+    structlogger.debug(
+        "server.routes", event_info=f"Available web server routes: \n{url_table}"
+    )
+
+    return output
+
+
+def extract_args(
+    kwargs: Dict[Text, Any], keys_to_extract: Set[Text]
+) -> Tuple[Dict[Text, Any], Dict[Text, Any]]:
+    """Go through the kwargs and filter out the specified keys.
+
+    Return both, the filtered kwargs as well as the remaining kwargs.
+    """
+    remaining = {}
+    extracted = {}
+    for k, v in kwargs.items():
+        if k in keys_to_extract:
+            extracted[k] = v
+        else:
+            remaining[k] = v
+
+    return extracted, remaining
+
+
+def is_limit_reached(num_messages: int, limit: Optional[int]) -> bool:
+    """Determine whether the number of messages has reached a limit.
+
+    Args:
+        num_messages: The number of messages to check.
+        limit: Limit on the number of messages.
+
+    Returns:
+        `True` if the limit has been reached, otherwise `False`.
+    """
+    return limit is not None and num_messages >= limit
+
+
+class AvailableEndpoints:
+    """Collection of configured endpoints."""
+
+    _instance = None
+
+    @classmethod
+    def read_endpoints(cls, endpoint_file: Text) -> "AvailableEndpoints":
+        """Read the different endpoints from a yaml file."""
+        nlg = read_endpoint_config(endpoint_file, endpoint_type="nlg")
+        nlu = read_endpoint_config(endpoint_file, endpoint_type="nlu")
+        action = read_endpoint_config(endpoint_file, endpoint_type="action_endpoint")
+        model = read_endpoint_config(endpoint_file, endpoint_type="models")
+        tracker_store = read_endpoint_config(
+            endpoint_file, endpoint_type="tracker_store"
+        )
+        lock_store = read_endpoint_config(endpoint_file, endpoint_type="lock_store")
+        event_broker = read_endpoint_config(endpoint_file, endpoint_type="event_broker")
+        vector_store = read_endpoint_config(endpoint_file, endpoint_type="vector_store")
+        model_groups = read_property_config_from_endpoints_file(
+            endpoint_file, property_name="model_groups"
+        )
+
+        return cls(
+            nlg,
+            nlu,
+            action,
+            model,
+            tracker_store,
+            lock_store,
+            event_broker,
+            vector_store,
+            model_groups,
+        )
+
+    def __init__(
+        self,
+        nlg: Optional[EndpointConfig] = None,
+        nlu: Optional[EndpointConfig] = None,
+        action: Optional[EndpointConfig] = None,
+        model: Optional[EndpointConfig] = None,
+        tracker_store: Optional[EndpointConfig] = None,
+        lock_store: Optional[EndpointConfig] = None,
+        event_broker: Optional[EndpointConfig] = None,
+        vector_store: Optional[EndpointConfig] = None,
+        model_groups: Optional[List[Dict[str, Any]]] = None,
+    ) -> None:
+        """Create an `AvailableEndpoints` object."""
+        self.model = model
+        self.action = action
+        self.nlu = nlu
+        self.nlg = nlg
+        self.tracker_store = tracker_store
+        self.lock_store = lock_store
+        self.event_broker = event_broker
+        self.vector_store = vector_store
+        self.model_groups = model_groups
+
+    @classmethod
+    def get_instance(
+        cls, endpoint_file: Optional[Text] = DEFAULT_ENDPOINTS_PATH
+    ) -> "AvailableEndpoints":
+        """Get the singleton instance of AvailableEndpoints."""
+        # Ensure that the instance is initialized only once.
+        if cls._instance is None:
+            cls._instance = cls.read_endpoints(endpoint_file)
+        return cls._instance
+
+
+def read_endpoints_from_path(
+    endpoints_path: Optional[Union[Path, Text]] = None,
+) -> AvailableEndpoints:
+    """Get `AvailableEndpoints` object from specified path.
+
+    Args:
+        endpoints_path: Path of the endpoints file to be read. If `None` the
+            default path for that file is used (`endpoints.yml`).
+
+    Returns:
+        `AvailableEndpoints` object read from endpoints file.
+
+    """
+    endpoints_config_path = cli_utils.get_validated_path(
+        endpoints_path, "endpoints", DEFAULT_ENDPOINTS_PATH, True
+    )
+    return AvailableEndpoints.get_instance(endpoints_config_path)
+
+
+def _lock_store_is_multi_worker_compatible(
+    lock_store: Union[EndpointConfig, LockStore, None],
+) -> bool:
+    if isinstance(lock_store, InMemoryLockStore):
+        return False
+
+    if isinstance(lock_store, RedisLockStore):
+        return True
+
+    # `lock_store` is `None` or `EndpointConfig`
+    return (
+        lock_store is not None
+        and isinstance(lock_store, EndpointConfig)
+        and lock_store.type != "in_memory"
+    )
+
+
+def number_of_sanic_workers(lock_store: Union[EndpointConfig, LockStore, None]) -> int:
+    """Get the number of Sanic workers to use in `app.run()`.
+
+    If the environment variable constants.ENV_SANIC_WORKERS is set and is not equal to
+    1, that value will only be permitted if the used lock store is not the
+    `InMemoryLockStore`.
+    """
+
+    def _log_and_get_default_number_of_workers() -> int:
+        structlogger.debug(
+            "server.worker.set_count",
+            number_of_workers=DEFAULT_SANIC_WORKERS,
+            event_info=f"Using the default number of Sanic workers "
+            f"({DEFAULT_SANIC_WORKERS}).",
+        )
+        return DEFAULT_SANIC_WORKERS
+
+    try:
+        env_value = int(os.environ.get(ENV_SANIC_WORKERS, DEFAULT_SANIC_WORKERS))
+    except ValueError:
+        structlogger.error(
+            "server.worker.set_count.error",
+            number_of_workers=os.environ[ENV_SANIC_WORKERS],
+            event_info=f"Cannot convert environment variable `{ENV_SANIC_WORKERS}` "
+            f"to int ('{os.environ[ENV_SANIC_WORKERS]}').",
+        )
+        return _log_and_get_default_number_of_workers()
+
+    if env_value == DEFAULT_SANIC_WORKERS:
+        return _log_and_get_default_number_of_workers()
+
+    if env_value < 1:
+        structlogger.warning(
+            "server.worker.set_count.error_less_than_one",
+            number_of_workers=env_value,
+            event_info=f"Cannot set number of Sanic workers to the desired value "
+            f"({env_value}). The number of workers must be at least 1.",
+        )
+        return _log_and_get_default_number_of_workers()
+
+    if _lock_store_is_multi_worker_compatible(lock_store):
+        structlogger.debug(
+            "server.worker.set_count.success",
+            event_info=f"Using {env_value} Sanic workers.",
+            num_workers=env_value,
+        )
+        return env_value
+
+    structlogger.warning(
+        "server.worker.set_count.error_no_lock_store",
+        event_info=f"Unable to assign desired number of Sanic workers ({env_value}) as "
+        f"no `RedisLockStore` or custom `LockStore` endpoint "
+        f"configuration has been found.",
+        num_workers=env_value,
+    )
+    return _log_and_get_default_number_of_workers()
+
+
+def add_bot_utterance_metadata(
+    message: Dict[str, Any],
+    domain_response_name: str,
+    nlg: "NaturalLanguageGenerator",
+    domain: "Domain",
+    tracker: Optional[DialogueStateTracker],
+) -> Dict[str, Any]:
+    """Add metadata to the bot message."""
+    message["utter_action"] = domain_response_name
+
+    utter_source = message.get(UTTER_SOURCE_METADATA_KEY)
+    if utter_source is None:
+        utter_source = nlg.__class__.__name__
+        message[UTTER_SOURCE_METADATA_KEY] = utter_source
+
+    if tracker:
+        message[ACTIVE_FLOW_METADATA_KEY] = tracker.active_flow
+        message[STEP_ID_METADATA_KEY] = tracker.current_step_id
+
+    if utter_source in ["IntentlessPolicy", "ContextualResponseRephraser"]:
+        message[DOMAIN_GROUND_TRUTH_METADATA_KEY] = [
+            response.get("text")
+            for response in domain.responses.get(domain_response_name, [])
+            if response.get("text") is not None
+        ]
+
+    return message

rasa/core/visualize.py

@@ -0,0 +1,70 @@
+import logging
+import os
+from typing import Text
+
+from rasa import telemetry
+from rasa.shared.core.training_data import loading
+from rasa.shared.utils.cli import print_error
+from rasa.shared.core.domain import InvalidDomain, Domain
+
+logger = logging.getLogger(__name__)
+
+
+def visualize(
+    domain_path: Text,
+    stories_path: Text,
+    nlu_data_path: Text,
+    output_path: Text,
+    max_history: int,
+) -> None:
+    """Visualizes stories as graph.
+
+    Args:
+        domain_path: Path to the domain file.
+        stories_path: Path to the stories files.
+        nlu_data_path: Path to the NLU training data which can be used to interpolate
+            intents with actual examples in the graph.
+        output_path: Path where the created graph should be persisted.
+        max_history: Max history to use for the story visualization.
+    """
+    import rasa.shared.core.training_data.visualization
+
+    try:
+        domain = Domain.load(domain_path)
+    except InvalidDomain as e:
+        print_error(
+            f"Could not load domain due to: '{e}'. To specify a valid domain path use "
+            f"the '--domain' argument."
+        )
+        return
+
+    # this is optional, only needed if the `/greet` type of
+    # messages in the stories should be replaced with actual
+    # messages (e.g. `hello`)
+    if nlu_data_path is not None:
+        import rasa.shared.nlu.training_data.loading
+
+        nlu_training_data = rasa.shared.nlu.training_data.loading.load_data(
+            nlu_data_path
+        )
+    else:
+        nlu_training_data = None
+
+    logger.info("Starting to visualize stories...")
+    telemetry.track_visualization()
+
+    story_steps = loading.load_data_from_resource(stories_path, domain)
+    rasa.shared.core.training_data.visualization.visualize_stories(
+        story_steps,
+        domain,
+        output_path,
+        max_history,
+        nlu_training_data=nlu_training_data,
+    )
+
+    full_output_path = "file://{}".format(os.path.abspath(output_path))
+    logger.info(f"Finished graph creation. Saved into {full_output_path}")
+
+    import webbrowser
+
+    webbrowser.open(full_output_path)

rasa/dialogue_understanding/coexistence/constants.py

@@ -0,0 +1,4 @@
+CALM_ENTRY = "calm_entry"
+NLU_ENTRY = "nlu_entry"
+STICKY = "sticky"
+NON_STICKY = "non_sticky"

rasa/dialogue_understanding/coexistence/intent_based_router.py

@@ -0,0 +1,196 @@
+from __future__ import annotations
+from typing import Any, Dict, List, Optional
+
+import structlog
+
+from rasa.dialogue_understanding.coexistence.constants import (
+    CALM_ENTRY,
+    NLU_ENTRY,
+    STICKY,
+    NON_STICKY,
+)
+from rasa.dialogue_understanding.commands import Command, SetSlotCommand
+from rasa.dialogue_understanding.commands.noop_command import NoopCommand
+from rasa.dialogue_understanding.generator.nlu_command_adapter import (
+    NLUCommandAdapter,
+)
+from rasa.engine.graph import ExecutionContext, GraphComponent
+from rasa.engine.recipes.default_recipe import DefaultV1Recipe
+from rasa.engine.storage.resource import Resource
+from rasa.engine.storage.storage import ModelStorage
+from rasa.shared.constants import ROUTE_TO_CALM_SLOT
+from rasa.shared.core.domain import Domain
+from rasa.shared.core.flows import FlowsList
+from rasa.shared.core.trackers import DialogueStateTracker
+from rasa.shared.exceptions import InvalidConfigException
+from rasa.shared.nlu.constants import COMMANDS, INTENT, INTENT_NAME_KEY
+from rasa.shared.nlu.training_data.message import Message
+from rasa.shared.nlu.training_data.training_data import TrainingData
+
+
+structlogger = structlog.get_logger()
+
+
+@DefaultV1Recipe.register(
+    [
+        DefaultV1Recipe.ComponentType.COEXISTENCE_ROUTER,
+    ],
+    is_trainable=False,
+)
+class IntentBasedRouter(GraphComponent):
+    @staticmethod
+    def get_default_config() -> Dict[str, Any]:
+        """The component's default config (see parent class for full docstring)."""
+        return {
+            NLU_ENTRY: None,
+            CALM_ENTRY: None,
+        }
+
+    def __init__(
+        self,
+        config: Dict[str, Any],
+        model_storage: ModelStorage,
+        resource: Resource,
+    ) -> None:
+        self.config = {**self.get_default_config(), **config}
+        self._model_storage = model_storage
+        self._resource = resource
+        self.validate_config()
+
+    def validate_config(self) -> None:
+        """Validate the config of the router."""
+        if not (self._is_nlu_entry_valid() and self._is_calm_entry_valid()):
+            raise ValueError(
+                "The IntentBasedRouter component needs a proper "
+                "description of the capabilities implemented in the DM1 "
+                "part of the bot."
+            )
+
+    def _is_nlu_entry_valid(self) -> bool:
+        """Check if the NLU entry in the config is valid."""
+        nlu_entry = self.config.get(NLU_ENTRY, {})
+        return (
+            isinstance(nlu_entry, dict)
+            and nlu_entry.get(STICKY) is not None
+            and nlu_entry.get(NON_STICKY) is not None
+        )
+
+    def _is_calm_entry_valid(self) -> bool:
+        """Check if the CALM entry in the config is valid."""
+        calm_entry = self.config.get(CALM_ENTRY, {})
+        return isinstance(calm_entry, dict) and calm_entry.get(STICKY) is not None
+
+    def train(self, training_data: TrainingData) -> Resource:
+        """Train the intent classifier on a data set."""
+        return self._resource
+
+    @classmethod
+    def load(
+        cls,
+        config: Dict[str, Any],
+        model_storage: ModelStorage,
+        resource: Resource,
+        execution_context: ExecutionContext,
+        **kwargs: Any,
+    ) -> "IntentBasedRouter":
+        """Loads trained component (see parent class for full docstring)."""
+        return cls(config, model_storage, resource)
+
+    @classmethod
+    def create(
+        cls,
+        config: Dict[str, Any],
+        model_storage: ModelStorage,
+        resource: Resource,
+        execution_context: ExecutionContext,
+    ) -> IntentBasedRouter:
+        """Creates component (see parent class for full docstring)."""
+        return cls(config, model_storage, resource)
+
+    async def process(
+        self,
+        messages: List[Message],
+        flows: FlowsList,
+        tracker: Optional[DialogueStateTracker] = None,
+        domain: Optional[Domain] = None,
+    ) -> List[Message]:
+        """Process a list of messages."""
+        if tracker is None:
+            # cannot do anything if there is no tracker (happens during 'rasa test nlu')
+            return messages
+
+        for message in messages:
+            commands = await self.predict_commands(message, flows, tracker, domain)
+            commands_dicts = [command.as_dict() for command in commands]
+            message.set(COMMANDS, commands_dicts, add_to_output=True)
+
+        return messages
+
+    async def predict_commands(
+        self,
+        message: Message,
+        flows: FlowsList,
+        tracker: DialogueStateTracker,
+        domain: Optional[Domain] = None,
+    ) -> List[Command]:
+        if not tracker.has_coexistence_routing_slot:
+            raise InvalidConfigException(
+                f"Tried to run the IntentBasedRouter component "
+                f"without the slot to track coexistence routing ({ROUTE_TO_CALM_SLOT})."
+            )
+
+        route_session_to_calm = tracker.get_slot(ROUTE_TO_CALM_SLOT)
+        if route_session_to_calm is None:
+            commands = self._generate_command_using_intent(message, flows, tracker)
+            structlogger.info(
+                "intent_based_router.predicated_commands", commands=commands
+            )
+            return commands
+        elif route_session_to_calm is True:
+            # don't set any commands so that a `LLMBasedCommandGenerator` is
+            # triggered and can predict the actual commands.
+            return []
+        else:
+            # If the session is assigned to DM1 add a `NoopCommand` to silence
+            # the other command generators.
+            return [NoopCommand()]
+
+    def _check_intent_part_of_nlu_trigger(
+        self, message: Message, tracker: DialogueStateTracker, flows: FlowsList
+    ) -> bool:
+        """Check if the intent is part of a nlu trigger."""
+        commands = NLUCommandAdapter.convert_nlu_to_commands(
+            message,
+            tracker,
+            flows,
+        )
+        return len(commands) > 0
+
+    def _generate_command_using_intent(
+        self, message: Message, flows: FlowsList, tracker: DialogueStateTracker
+    ) -> List[Command]:
+        intent = message.data.get(INTENT)
+        if not intent or intent.get(INTENT_NAME_KEY) is None:
+            # If the message does not have a predicted intent, let the session
+            # be sticky and routed to DM1. This is done as the customers currently
+            # are in the inital phase of migration to CALM. In the future, we can
+            # let the next command generator predict the commands.
+            return [SetSlotCommand(ROUTE_TO_CALM_SLOT, False)]
+
+        if intent[INTENT_NAME_KEY] in self.config[NLU_ENTRY][STICKY]:
+            # If the intent is in nlu entry sticky, set the slot to route to DM1.
+            return [SetSlotCommand(ROUTE_TO_CALM_SLOT, False)]
+        elif intent[INTENT_NAME_KEY] in self.config[NLU_ENTRY][NON_STICKY]:
+            # If the intent is in nlu entry non-sticky, predict a Noop so that
+            # the next command generator is skipped for just this turn.
+            return [NoopCommand()]
+        elif intent[INTENT_NAME_KEY] in self.config[CALM_ENTRY][STICKY]:
+            # If the intent is in calm entry sticky, set the slot to route to CALM.
+            return []
+
+        if self._check_intent_part_of_nlu_trigger(message, tracker, flows):
+            # If the intent is part of a nlu trigger, set the slot to route to CALM.
+            return []
+        # If the intent is not present in any of the above, let the session be sticky
+        # and routed to DM1.
+        return [SetSlotCommand(ROUTE_TO_CALM_SLOT, False)]