rasa-pro 3.12.0.dev1__py3-none-any.whl

rasa/utils/log_utils.py

@@ -0,0 +1,181 @@
+from __future__ import annotations
+import os
+import logging
+import sys
+from typing import Any, Dict, Optional
+
+import structlog
+from structlog_sentry import SentryProcessor
+from structlog.dev import ConsoleRenderer
+from structlog.typing import EventDict, WrappedLogger
+from rasa.shared.constants import (
+    ENV_LOG_LEVEL,
+    DEFAULT_LOG_LEVEL,
+    ENV_LOG_LEVEL_LLM,
+    ENV_LOG_LEVEL_LLM_MODULE_NAMES,
+    DEFAULT_LOG_LEVEL_LLM,
+)
+from rasa.plugin import plugin_manager
+
+
+FORCE_JSON_LOGGING = os.environ.get("FORCE_JSON_LOGGING")
+
+
+class HumanConsoleRenderer(ConsoleRenderer):
+    """Console renderer that outputs human-readable logs."""
+
+    def __call__(self, logger: WrappedLogger, name: str, event_dict: EventDict) -> str:
+        if "event_info" in event_dict:
+            event_key = event_dict["event"]
+            event_dict["event"] = event_dict["event_info"]
+            event_dict["event_key"] = event_key
+            del event_dict["event_info"]
+
+        return super().__call__(logger, name, event_dict)
+
+
+def _anonymizer(
+    _: structlog.BoundLogger, __: str, event_dict: Dict[str, Any]
+) -> Dict[str, Any]:
+    """Anonymizes event dict."""
+    anonymizable_keys = [
+        "text",
+        "response_text",
+        "user_text",
+        "slots",
+        "parse_data_text",
+        "parse_data_entities",
+        "prediction_events",
+        "tracker_latest_message",
+        "prefilled_slots",
+        "message",
+        "response",
+        "slot_candidates",
+        "rasa_event",
+        "rasa_events",
+        "tracker_states",
+        "current_states",
+        "old_states",
+        "current_states",
+        "successes",
+        "current_entity",
+        "next_entity",
+        "states",
+        "entity",
+        "token_text",
+        "user_message",
+        "json_message",
+    ]
+    anonymization_pipeline = plugin_manager().hook.get_anonymization_pipeline()
+
+    if anonymization_pipeline:
+        for key in anonymizable_keys:
+            if key in event_dict:
+                anonymized_value = anonymization_pipeline.log_run(event_dict[key])
+                event_dict[key] = anonymized_value
+    return event_dict
+
+
+def configure_structlog(
+    log_level: Optional[int] = None,
+    include_time: bool = False,
+) -> None:
+    """Configure logging of the server."""
+    if log_level is None:  # Log level NOTSET is 0 so we use `is None` here
+        log_level_name = os.environ.get(ENV_LOG_LEVEL, DEFAULT_LOG_LEVEL)
+        # Change log level from str to int (note that log_level in function parameter
+        # int already, coming from CLI argparse parameter).
+        log_level = logging.getLevelName(log_level_name)
+
+    logging.basicConfig(
+        format="%(message)s",
+        stream=sys.stdout,
+        level=log_level,
+    )
+
+    shared_processors = [
+        _anonymizer,
+        # Processors that have nothing to do with output,
+        # e.g., add timestamps or log level names.
+        # If log level is too low, abort pipeline and throw away log entry.
+        structlog.stdlib.filter_by_level,
+        structlog.contextvars.merge_contextvars,
+        # Add the name of the logger to event dict.
+        # structlog.stdlib.add_logger_name,
+        # Add log level to event dict.
+        structlog.processors.add_log_level,
+        # If the "stack_info" key in the event dict is true, remove it and
+        # render the current stack trace in the "stack" key.
+        structlog.processors.StackInfoRenderer(),
+        # If some value is in bytes, decode it to a unicode str.
+        structlog.processors.UnicodeDecoder(),
+        structlog.dev.set_exc_info,
+        # add structlog sentry integration. only log fatal log entries
+        # as events as we are tracking exceptions anyways
+        SentryProcessor(event_level=logging.FATAL),
+    ]
+
+    if include_time:
+        shared_processors.append(structlog.processors.TimeStamper(fmt="iso"))
+
+    if not FORCE_JSON_LOGGING and sys.stderr.isatty():
+        # Pretty printing when we run in a terminal session.
+        # Automatically prints pretty tracebacks when "rich" is installed
+        processors = shared_processors + [
+            HumanConsoleRenderer(),
+        ]
+    else:
+        # Print JSON when we run, e.g., in a Docker container.
+        # Also print structured tracebacks.
+        processors = shared_processors + [
+            structlog.processors.dict_tracebacks,
+            structlog.processors.JSONRenderer(),
+        ]
+
+    structlog.configure(
+        processors=processors,  # type: ignore
+        context_class=dict,
+        # `logger_factory` is used to create wrapped loggers that are used for
+        # OUTPUT. This one returns a `logging.Logger`. The final value (a JSON
+        # string) from the final processor (`JSONRenderer`) will be passed to
+        # the method of the same name as that you've called on the bound logger.
+        logger_factory=structlog.stdlib.LoggerFactory(),
+        # `wrapper_class` is the bound logger that you get back from
+        # get_logger(). This one imitates the API of `logging.Logger`.
+        wrapper_class=structlog.make_filtering_bound_logger(log_level),
+        # Effectively freeze configuration after creating the first bound
+        # logger.
+        cache_logger_on_first_use=True,
+    )
+    # doing logger creation inline, to prevent usage of unconfigured logger
+    structlog.get_logger().debug("structlog.configured")
+
+
+def log_llm(logger: Any, log_module: str, log_event: str, **kwargs: Any) -> None:
+    """Logs LLM-specific events depending on a flag passed through an env var.
+
+    If the module's flag is set to INFO (e.g.
+    LOG_PROMPT_LLM_COMMAND_GENERATOR=INFO), its prompt is logged at INFO level,
+    overriding the general log level setting.
+
+    Args:
+        logger: instance of the structlogger of the component
+        log_module: name of the module/component logging the event
+        log_event: string describing the log event
+        **kwargs: dictionary of additional logging context
+    """
+    log_level_llm_name = os.environ.get(ENV_LOG_LEVEL_LLM, DEFAULT_LOG_LEVEL_LLM)
+    log_level_llm = logging.getLevelName(log_level_llm_name.upper())
+
+    module_env_variable = ENV_LOG_LEVEL_LLM_MODULE_NAMES.get(
+        log_module, "LOG_LEVEL_LLM_" + log_module.upper()
+    )
+    log_level_llm_module_name = os.environ.get(
+        module_env_variable, DEFAULT_LOG_LEVEL_LLM
+    )
+    log_level_llm_module = logging.getLevelName(log_level_llm_module_name.upper())
+
+    # log at the highest specified level, e.g. max(DEBUG=10, INFO=20)
+    log_level = max(log_level_llm, log_level_llm_module)
+
+    logger.log(log_level, log_event, **kwargs)

rasa/utils/mapper.py

@@ -0,0 +1,210 @@
+from pathlib import Path
+from typing import Dict, List, Optional, Text, Union
+
+import rasa.shared.data
+import rasa.shared.utils.io
+from rasa.shared.core.domain import (
+    KEY_ACTIONS,
+    KEY_ENTITIES,
+    KEY_FORMS,
+    KEY_INTENTS,
+    KEY_RESPONSES,
+    KEY_SLOTS,
+    Domain,
+)
+from rasa.shared.core.flows.yaml_flows_io import KEY_FLOWS, is_flows_file
+from rasa.shared.core.training_data.story_reader.yaml_story_reader import (
+    KEY_RULE_NAME,
+    KEY_RULES,
+    KEY_STORIES,
+    KEY_STORY_NAME,
+    YAMLStoryReader,
+)
+from rasa.shared.nlu.training_data.formats.rasa_yaml import (
+    KEY_INTENT,
+    KEY_NLU,
+)
+from rasa.shared.utils.yaml import read_yaml_file
+
+
+class RasaPrimitiveStorageMapper:
+    """Maps the Rasa primitives to the file its located in."""
+
+    def __init__(
+        self,
+        domain_path: Optional[Union[Text, Path]] = None,
+        training_data_paths: Optional[Union[List[Text], List[Path], Text, Path]] = None,
+    ):
+        self._primitives: Dict[str, Dict] = {
+            "entities": {},
+            "slots": {},
+            "forms": {},
+            "intents": {},
+            "stories": {},
+            "rules": {},
+            "actions": {},
+            "responses": {},
+            "flows": {},
+        }
+
+        if domain_path:
+            self._load_domain(domain_path)
+
+        if training_data_paths:
+            self._load_training_data(training_data_paths)
+
+    def _load_domain(self, domain_path: Union[Text, Path]) -> None:
+        self._resolve_domain_files(domain_path)
+        self._parse_domain_files()
+
+    def _load_training_data(
+        self, training_data_paths: Union[List[Text], List[Path], Text, Path]
+    ) -> None:
+        self._resolve_training_data_files(training_data_paths)
+
+        self._nlu_files = rasa.shared.data.get_data_files(
+            self._training_data_paths, rasa.shared.data.is_nlu_file
+        )
+        self._parse_nlu_data()
+
+        self._story_files = rasa.shared.data.get_data_files(
+            self._training_data_paths, YAMLStoryReader.is_stories_file
+        )
+        self._parse_story_data()
+
+        self._flow_files = rasa.shared.data.get_data_files(
+            self._training_data_paths, is_flows_file
+        )
+        self._parse_flow_data()
+
+    def _resolve_training_data_files(
+        self, training_data_paths: Union[List[Text], List[Path], Text, Path]
+    ) -> None:
+        if isinstance(training_data_paths, list):
+            training_data_paths = [Path(p) for p in training_data_paths]
+        elif isinstance(training_data_paths, str):
+            training_data_paths = [Path(training_data_paths)]
+        elif isinstance(training_data_paths, Path):
+            training_data_paths = [training_data_paths]
+
+        self._training_data_paths = training_data_paths
+
+    def _resolve_domain_files(self, domain_path: Union[Text, Path]) -> None:
+        domain_path = Path(domain_path) if isinstance(domain_path, str) else domain_path
+
+        if domain_path.is_dir():
+            self._domain_files = [
+                file for file in domain_path.iterdir() if Domain.is_domain_file(file)
+            ]
+        else:
+            self._domain_files = [domain_path]
+
+    def _parse_domain_files(self) -> None:
+        for domain_file in self._domain_files:
+            content = read_yaml_file(domain_file)
+            for entity in content.get(KEY_ENTITIES, []):  # type: ignore[union-attr]
+                self._assign_value(
+                    self._primitives["entities"], "domain", domain_file, entity
+                )
+            for intent in content.get(KEY_INTENTS, []):  # type: ignore[union-attr]
+                self._assign_value(
+                    self._primitives["intents"], "domain", domain_file, intent
+                )
+            for slot in content.get(KEY_SLOTS, []):  # type: ignore[union-attr]
+                self._assign_value(
+                    self._primitives["slots"], "domain", domain_file, slot
+                )
+            for form in content.get(KEY_FORMS, []):  # type: ignore[union-attr]
+                self._assign_value(
+                    self._primitives["forms"], "domain", domain_file, form
+                )
+            for action in content.get(KEY_ACTIONS, []):  # type: ignore[union-attr]
+                self._assign_value(
+                    self._primitives["actions"], "domain", domain_file, action
+                )
+            for response in content.get(KEY_RESPONSES, []):  # type: ignore[union-attr]
+                self._assign_value(
+                    self._primitives["responses"], "domain", domain_file, response
+                )
+
+    def _parse_nlu_data(self) -> None:
+        """Parses the nlu data and extracts the intents."""
+        for nlu_file in self._nlu_files:
+            content = read_yaml_file(nlu_file)
+            for intent in content.get(KEY_NLU, []):  # type: ignore[union-attr]
+                if KEY_INTENT in intent:
+                    self._assign_value(
+                        self._primitives["intents"],
+                        "training",
+                        nlu_file,
+                        intent.get(KEY_INTENT),
+                    )
+
+    def _parse_story_data(self) -> None:
+        """Parses the story data and extracts the stories and rules."""
+        for story_file in self._story_files:
+            content = read_yaml_file(story_file)
+            for story in content.get(KEY_STORIES, []):  # type: ignore[union-attr]
+                self._assign_value(
+                    self._primitives["stories"],
+                    "training",
+                    story_file,
+                    story[KEY_STORY_NAME],
+                )
+            for rule in content.get(KEY_RULES, []):  # type: ignore[union-attr]
+                self._assign_value(
+                    self._primitives["rules"],
+                    "training",
+                    story_file,
+                    rule[KEY_RULE_NAME],
+                )
+
+    def _parse_flow_data(self) -> None:
+        """Parses the flow data and extracts the flow ids."""
+        for flow_file in self._flow_files:
+            content = read_yaml_file(flow_file)
+            for flow in content.get(KEY_FLOWS, []):  # type: ignore[union-attr]
+                self._assign_value(
+                    self._primitives["flows"],
+                    "training",
+                    flow_file,
+                    flow,
+                )
+
+    @staticmethod
+    def _assign_value(
+        primitive: Dict, file_type: str, value: Union[str, Path], key: str
+    ) -> None:
+        if not isinstance(value, Path):
+            value = Path(value)
+
+        if isinstance(key, dict):
+            key = next(iter(key.keys()))
+
+        if key in primitive:
+            if file_type in primitive[key]:
+                primitive[key][file_type].append(value)
+            else:
+                primitive[key][file_type] = [value]
+        else:
+            primitive[key] = {file_type: [value]}
+
+    def get_file(self, primitive: Text, primitive_type: Text) -> Dict[str, List[Path]]:
+        """Returns the file where the primitive is located.
+
+        If the primitive is not found, empty dict is returned.
+
+        Args:
+            primitive: The name or ID of the primitive to search for.
+            primitive_type: The type of the primitive to search for.
+                either entities, slots, forms, intents, stories or rules.
+
+        Returns:
+            A dictionary containing the file type and the list of file paths.
+            file_type can be either domain or training.
+            get_file()[file_type] -> List[Path]
+        """
+        try:
+            return self._primitives[primitive_type][primitive]
+        except KeyError:
+            return {}

rasa/utils/ml_utils.py

@@ -0,0 +1,147 @@
+from pathlib import Path
+from typing import Any, Dict, List, Optional, Text
+
+import numpy as np
+import structlog
+from langchain.schema.embeddings import Embeddings
+from langchain_community.vectorstores import FAISS
+from rasa.shared.constants import REQUIRED_SLOTS_KEY
+from rasa.shared.core.domain import KEY_RESPONSES_TEXT, Domain
+from rasa.shared.utils.llm import AI
+
+structlogger = structlog.get_logger()
+
+
+def persist_faiss_vector_store(path: Path, vecor_store: Optional[FAISS]) -> None:
+    """Persists the given vector store to the given path.
+
+    Args:
+        path: path to persist the vector store to
+        vecor_store: vector store to persist
+    """
+    if vecor_store:
+        vecor_store.save_local(str(path))
+
+
+def load_faiss_vector_store(path: Path, embedder: Embeddings) -> Optional[FAISS]:
+    """Loads the vector store from the given path.
+
+    Args:
+        path: path to load the vector store from
+        embedder: embedder to use for the vector store
+
+    Returns:
+        The loaded vector store or None if the path does not exist.
+    """
+    if path.exists():
+        return FAISS.load_local(
+            str(path), embedder, allow_dangerous_deserialization=True
+        )
+    else:
+        return None
+
+
+def extract_participant_messages_from_transcript(
+    transcript: Text, participant: Text = AI
+) -> List[Text]:
+    r"""Extracts repsonses from the transcript.
+
+    Args:
+        transcript: transcript text to process
+        participant: participant prefix (AI or Human, AI by default)
+
+    Example:
+        >>> extract_participant_messages_from_transcript("USER: Hello\\nAI: Hi there!")
+        ["Hi there!"]
+
+    Returns:
+        List of extracted responses.
+    """
+    prefix = participant + ": "
+    return [
+        line[len(prefix) :]
+        for line in transcript.split("\n")
+        if line.startswith(prefix)
+    ]
+
+
+def extract_ai_response_examples(
+    responses: Dict[Text, List[Dict[Text, Any]]],
+) -> List[str]:
+    """Extracts the responses from the domain.
+
+    Only responses that have a text field are extracted.
+
+    Example:
+        >>> responses = {
+        ...     "utter_greet": [
+        ...         {"text": "hello"},
+        ...     ],
+        ...     "utter_goodbye": [
+        ...         {"text": "goodbye"},
+        ...         {"text": ""},
+        ...     ],
+        ... }
+        >>> extract_ai_response_examples(responses)
+        ["hello", "goodbye"]
+
+    Args:
+        responses: the responses from the domain
+
+    Returns:
+    A list of responses.
+    """
+    example_ai_responses = []
+
+    for variations in responses.values():
+        if not variations:
+            continue
+        for variation in variations:
+            if response_text := variation.get(KEY_RESPONSES_TEXT):
+                example_ai_responses.append(response_text)
+
+    return example_ai_responses
+
+
+def response_for_template(
+    template_name: str, responses: Dict[Text, List[Dict[Text, Any]]]
+) -> Optional[str]:
+    """Returns an interaction for the given template name from the responses.
+
+    Args:
+        template_name: the name of the template
+        responses: the responses from the domain
+
+    Returns:
+    The response text or None if no response was found.
+    """
+    rsps = responses.get(template_name, [])
+    usuable_responses = [r for r in rsps if r.get(KEY_RESPONSES_TEXT)]
+    if usuable_responses:
+        selected_response = np.random.choice(usuable_responses)  # type: ignore
+        return selected_response.get(KEY_RESPONSES_TEXT)
+    else:
+        structlogger.warning(
+            "response_template.not_found",
+            template_name=template_name,
+        )
+        return None
+
+
+def form_utterances_to_action(domain: Domain) -> Dict[str, str]:
+    """Returns a dictionary of form utterances to their form name.
+
+    Args:
+        domain: the domain of the assistant
+
+
+    Returns:
+    A dictionary of form utterances to their form name.
+    """
+    form_slots = [
+        (slot_name, form_name)
+        for form_name, props in domain.forms.items()
+        for slot_name in props.get(REQUIRED_SLOTS_KEY, [])
+    ]
+
+    return {f"utter_ask_{slot_name}": form_name for slot_name, form_name in form_slots}