rasa-pro 3.8.16__py3-none-any.whl

rasa/tracing/instrumentation/intentless_policy_instrumentation.py

@@ -0,0 +1,142 @@
+import functools
+import json
+import logging
+from typing import Callable, List, Optional, TYPE_CHECKING, Type
+
+from opentelemetry.trace import Tracer
+
+if TYPE_CHECKING:
+    from rasa.core.policies.intentless_policy import IntentlessPolicy
+
+logger = logging.getLogger(__name__)
+
+
+def _instrument_select_response_examples(
+    tracer: Tracer, policy_class: Type["IntentlessPolicy"]
+) -> None:
+    def tracing_select_response_examples_wrapper(fn: Callable) -> Callable:
+        @functools.wraps(fn)
+        def wrapper(
+            self: "IntentlessPolicy",
+            history: str,
+            number_of_samples: int,
+            max_number_of_tokens: int,
+        ) -> List[str]:
+            with tracer.start_as_current_span(
+                f"{self.__class__.__name__}.{fn.__name__}"
+            ) as span:
+                ai_response_examples = fn(
+                    self, history, number_of_samples, max_number_of_tokens
+                )
+                span.set_attributes(
+                    {
+                        "ai_response_examples": json.dumps(ai_response_examples),
+                    }
+                )
+                return ai_response_examples
+
+        return wrapper
+
+    policy_class.select_response_examples = tracing_select_response_examples_wrapper(  # type: ignore[assignment]  # noqa: E501
+        policy_class.select_response_examples
+    )
+
+    logger.debug(
+        f"Instrumented '{policy_class.__name__}.select_response_examples' method."
+    )
+
+
+def _instrument_select_few_shot_conversations(
+    tracer: Tracer, policy_class: Type["IntentlessPolicy"]
+) -> None:
+    def tracing_select_few_shot_conversations_wrapper(fn: Callable) -> Callable:
+        @functools.wraps(fn)
+        def wrapper(
+            self: "IntentlessPolicy",
+            history: str,
+            number_of_samples: int,
+            max_number_of_tokens: int,
+        ) -> List[str]:
+            with tracer.start_as_current_span(
+                f"{self.__class__.__name__}.{fn.__name__}"
+            ) as span:
+                conversation_samples = fn(
+                    self, history, number_of_samples, max_number_of_tokens
+                )
+                span.set_attributes(
+                    {
+                        "conversation_samples": json.dumps(conversation_samples),
+                    }
+                )
+                return conversation_samples
+
+        return wrapper
+
+    policy_class.select_few_shot_conversations = tracing_select_few_shot_conversations_wrapper(  # type: ignore[assignment]  # noqa: E501
+        policy_class.select_few_shot_conversations
+    )
+
+    logger.debug(
+        f"Instrumented '{policy_class.__name__}.select_few_shot_conversations' method."
+    )
+
+
+def _instrument_extract_ai_responses(
+    tracer: Tracer, policy_class: Type["IntentlessPolicy"]
+) -> None:
+    def tracing_extract_ai_responses_wrapper(fn: Callable) -> Callable:
+        @functools.wraps(fn)
+        def wrapper(
+            self: "IntentlessPolicy", conversation_samples: List[str]
+        ) -> List[str]:
+            with tracer.start_as_current_span(
+                f"{self.__class__.__name__}.{fn.__name__}"
+            ) as span:
+                ai_responses = fn(self, conversation_samples)
+                span.set_attributes(
+                    {
+                        "ai_responses": json.dumps(ai_responses),
+                    }
+                )
+                return ai_responses
+
+        return wrapper
+
+    policy_class.extract_ai_responses = tracing_extract_ai_responses_wrapper(  # type: ignore[assignment]  # noqa: E501
+        policy_class.extract_ai_responses
+    )
+
+    logger.debug(f"Instrumented '{policy_class.__name__}.extract_ai_responses' method.")
+
+
+def _instrument_generate_answer(
+    tracer: Tracer, policy_class: Type["IntentlessPolicy"]
+) -> None:
+    def tracing_generate_answer_wrapper(fn: Callable) -> Callable:
+        @functools.wraps(fn)
+        async def wrapper(
+            self: "IntentlessPolicy",
+            response_examples: List[str],
+            conversation_samples: List[str],
+            history: str,
+        ) -> Optional[str]:
+            with tracer.start_as_current_span(
+                f"{self.__class__.__name__}.{fn.__name__}"
+            ) as span:
+                llm_response = await fn(
+                    self, response_examples, conversation_samples, history
+                )
+                span.set_attributes(
+                    {
+                        "llm_response": json.dumps(llm_response),
+                    }
+                )
+                return llm_response
+
+        return wrapper
+
+    policy_class.generate_answer = tracing_generate_answer_wrapper(  # type: ignore[assignment]  # noqa: E501
+        policy_class.generate_answer
+    )
+
+    logger.debug(f"Instrumented '{policy_class.__name__}.generate_answer' method.")

rasa/tracing/instrumentation/metrics.py

@@ -0,0 +1,206 @@
+from typing import Dict, Any
+
+import psutil
+
+from rasa.core.nlg.contextual_response_rephraser import ContextualResponseRephraser
+from rasa.core.policies.enterprise_search_policy import EnterpriseSearchPolicy
+from rasa.core.policies.intentless_policy import IntentlessPolicy
+from rasa.dialogue_understanding.generator import LLMCommandGenerator
+from rasa.tracing.constants import (
+    LLM_COMMAND_GENERATOR_CPU_USAGE_METRIC_NAME,
+    LLM_COMMAND_GENERATOR_MEMORY_USAGE_METRIC_NAME,
+    LLM_COMMAND_GENERATOR_PROMPT_TOKEN_USAGE_METRIC_NAME,
+    PROMPT_TOKEN_LENGTH_ATTRIBUTE_NAME,
+    LLM_COMMAND_GENERATOR_LLM_RESPONSE_DURATION_METRIC_NAME,
+    ENTERPRISE_SEARCH_POLICY_LLM_RESPONSE_DURATION_METRIC_NAME,
+    INTENTLESS_POLICY_LLM_RESPONSE_DURATION_METRIC_NAME,
+    CONTEXTUAL_RESPONSE_REPHRASER_LLM_RESPONSE_DURATION_METRIC_NAME,
+    RASA_CLIENT_REQUEST_BODY_SIZE_METRIC_NAME,
+    RASA_CLIENT_REQUEST_DURATION_METRIC_NAME,
+    ENDPOINT_REQUEST_BODY_SIZE_IN_BYTES_ATTRIBUTE_NAME,
+)
+from rasa.tracing.metric_instrument_provider import MetricInstrumentProvider
+from rasa.utils.endpoints import EndpointConfig
+
+
+def record_llm_command_generator_cpu_usage(
+    metric_instrument_provider: MetricInstrumentProvider,
+) -> None:
+    """Record CPU usage as a percentage.
+
+    The recording is done by the opentelemetry.metrics.Histogram instrument.
+    This instrument is registered to the MetricInstrumentProvider internal singleton.
+
+    :param metric_instrument_provider: The MetricInstrumentProvider instance
+    :return: None
+    """
+    metric_instrument = metric_instrument_provider.get_instrument(
+        LLM_COMMAND_GENERATOR_CPU_USAGE_METRIC_NAME
+    )
+    if not metric_instrument:
+        return None
+
+    cpu_usage = psutil.cpu_percent()
+    metric_instrument.record(amount=cpu_usage)
+
+
+def record_llm_command_generator_memory_usage(
+    metric_instrument_provider: MetricInstrumentProvider,
+) -> None:
+    """Record memory usage as a percentage.
+
+    The recording is done by the opentelemetry.metrics.Histogram instrument.
+    This instrument is registered to the MetricInstrumentProvider internal singleton.
+
+    :param metric_instrument_provider: The MetricInstrumentProvider instance
+    :return: None
+    """
+    metric_instrument = metric_instrument_provider.get_instrument(
+        LLM_COMMAND_GENERATOR_MEMORY_USAGE_METRIC_NAME
+    )
+    if not metric_instrument:
+        return None
+
+    memory_usage = psutil.virtual_memory().percent
+    metric_instrument.record(amount=memory_usage)
+
+
+def record_llm_command_generator_prompt_token(
+    metric_instrument_provider: MetricInstrumentProvider,
+    attributes: Dict[str, Any],
+) -> None:
+    """
+    Record prompt token length.
+
+    The recording is done by the opentelemetry.metrics.Histogram instrument.
+    This instrument is registered to the MetricInstrumentProvider internal singleton.
+
+    :param metric_instrument_provider: The MetricInstrumentProvider instance
+    :param attributes: Extracted tracing attributes
+    :return: None
+    """
+    if PROMPT_TOKEN_LENGTH_ATTRIBUTE_NAME not in attributes:
+        return None
+
+    metric_instrument = metric_instrument_provider.get_instrument(
+        LLM_COMMAND_GENERATOR_PROMPT_TOKEN_USAGE_METRIC_NAME
+    )
+    if not metric_instrument:
+        return None
+
+    prompt_tokens_len = attributes[PROMPT_TOKEN_LENGTH_ATTRIBUTE_NAME]
+
+    try:
+        prompt_tokens_len = int(prompt_tokens_len)
+    except ValueError:
+        return None
+
+    metric_instrument.record(
+        amount=prompt_tokens_len,
+    )
+
+
+def record_llm_command_generator_metrics(attributes: Dict[str, Any]) -> None:
+    """
+    Record measurements for LLMCommandGenerator specific metrics.
+
+    The recording is done by the opentelemetry.metrics.Histogram instruments.
+    These instruments are registered to the MetricInstrumentProvider internal singleton.
+
+    :param attributes: Extracted tracing attributes
+    :return: None
+    """
+    instrument_provider = MetricInstrumentProvider()
+
+    if not instrument_provider.instruments:
+        return None
+
+    record_llm_command_generator_cpu_usage(instrument_provider)
+    record_llm_command_generator_memory_usage(instrument_provider)
+    record_llm_command_generator_prompt_token(instrument_provider, attributes)
+
+
+def record_callable_duration_metrics(
+    self: Any, start_time: int, end_time: int, **kwargs: Any
+) -> None:
+    """
+    Record duration of instrumented method calls invoked for the following components:
+    - LLMCommandGenerator
+    - EnterpriseSearchPolicy
+    - IntentlessPolicy
+    - ContextualResponseRephraser
+    - EndpointConfig
+
+    :param self: The instance on which the instrumented method is called on.
+    :param start_time: Start time measured by time.perf_counter_ns()
+    before the method is called.
+    :param end_time: End time measured by time.perf_counter_ns()
+    after the method is called.
+    :return: None
+    """
+    instrument_provider = MetricInstrumentProvider()
+    if not instrument_provider.instruments:
+        return None
+
+    metric_instrument = None
+    attributes = {}
+
+    if isinstance(self, LLMCommandGenerator):
+        metric_instrument = instrument_provider.get_instrument(
+            LLM_COMMAND_GENERATOR_LLM_RESPONSE_DURATION_METRIC_NAME
+        )
+
+    if isinstance(self, EnterpriseSearchPolicy):
+        metric_instrument = instrument_provider.get_instrument(
+            ENTERPRISE_SEARCH_POLICY_LLM_RESPONSE_DURATION_METRIC_NAME
+        )
+
+    if isinstance(self, IntentlessPolicy):
+        metric_instrument = instrument_provider.get_instrument(
+            INTENTLESS_POLICY_LLM_RESPONSE_DURATION_METRIC_NAME
+        )
+
+    if isinstance(self, ContextualResponseRephraser):
+        metric_instrument = instrument_provider.get_instrument(
+            CONTEXTUAL_RESPONSE_REPHRASER_LLM_RESPONSE_DURATION_METRIC_NAME
+        )
+
+    if isinstance(self, EndpointConfig):
+        metric_instrument = instrument_provider.get_instrument(
+            RASA_CLIENT_REQUEST_DURATION_METRIC_NAME
+        )
+        attributes = {"url": kwargs.get("url")}
+
+    if not metric_instrument:
+        return None
+
+    timedelta_ms = (end_time - start_time) / 1000000
+    metric_instrument.record(amount=timedelta_ms, attributes=attributes)
+
+
+def record_request_size_in_bytes(attributes: Dict[str, Any]) -> None:
+    """
+    Record endpoint request size in bytes for EndpointConfig class.
+
+    The recording is done by the opentelemetry.metrics.Histogram instruments.
+    These instruments are registered to the MetricInstrumentProvider internal singleton.
+
+    :param attributes: Extracted tracing attributes
+    :return: None
+    """
+
+    metric_instrument_provider = MetricInstrumentProvider()
+
+    if not metric_instrument_provider.instruments:
+        return None
+
+    metric_instrument = metric_instrument_provider.get_instrument(
+        RASA_CLIENT_REQUEST_BODY_SIZE_METRIC_NAME
+    )
+    if not metric_instrument:
+        return None
+
+    request_body_size = attributes.pop(
+        ENDPOINT_REQUEST_BODY_SIZE_IN_BYTES_ATTRIBUTE_NAME, 0
+    )
+    metric_instrument.record(amount=request_body_size, attributes=attributes)

rasa/tracing/metric_instrument_provider.py

@@ -0,0 +1,125 @@
+from typing import ClassVar, Dict, Any
+
+from opentelemetry.metrics import get_meter_provider
+from opentelemetry.sdk.metrics import Meter
+
+from rasa.utils.singleton import Singleton
+from rasa.tracing.constants import (
+    LLM_COMMAND_GENERATOR_CPU_USAGE_METRIC_NAME,
+    LLM_COMMAND_GENERATOR_MEMORY_USAGE_METRIC_NAME,
+    LLM_COMMAND_GENERATOR_PROMPT_TOKEN_USAGE_METRIC_NAME,
+    LLM_COMMAND_GENERATOR_LLM_RESPONSE_DURATION_METRIC_NAME,
+    LLM_COMMAND_GENERATOR_CPU_MEMORY_USAGE_UNIT_NAME,
+    DURATION_UNIT_NAME,
+    ENTERPRISE_SEARCH_POLICY_LLM_RESPONSE_DURATION_METRIC_NAME,
+    INTENTLESS_POLICY_LLM_RESPONSE_DURATION_METRIC_NAME,
+    CONTEXTUAL_RESPONSE_REPHRASER_LLM_RESPONSE_DURATION_METRIC_NAME,
+    RASA_CLIENT_REQUEST_DURATION_METRIC_NAME,
+    RASA_CLIENT_REQUEST_BODY_SIZE_METRIC_NAME,
+)
+
+
+class MetricInstrumentProvider(metaclass=Singleton):
+    """Singleton provider class of metric instruments."""
+
+    instruments: ClassVar[Dict[str, Any]] = {}
+
+    def register_instruments(self) -> None:
+        """Update instruments class attribute.
+
+        The registered instruments are subclasses of the
+        opentelemetry.metrics._internal.instrument.Instrument interface.
+        """
+        meter = get_meter_provider().get_meter(__name__)
+
+        instruments = {
+            **self._create_llm_command_generator_instruments(meter),
+            **self._create_llm_response_duration_instruments(meter),
+            **self._create_client_request_instruments(meter),
+        }
+
+        self.instruments.update(instruments)
+
+    def get_instrument(self, name: str) -> Any:
+        """Get the instrument mapped to the provided name."""
+        return self.instruments.get(name)
+
+    @staticmethod
+    def _create_llm_command_generator_instruments(meter: Meter) -> Dict[str, Any]:
+        llm_command_generator_cpu_usage = meter.create_histogram(
+            name=LLM_COMMAND_GENERATOR_CPU_USAGE_METRIC_NAME,
+            description="CPU percentage for LLMCommandGenerator",
+            unit=LLM_COMMAND_GENERATOR_CPU_MEMORY_USAGE_UNIT_NAME,
+        )
+
+        llm_command_generator_memory_usage = meter.create_histogram(
+            name=LLM_COMMAND_GENERATOR_MEMORY_USAGE_METRIC_NAME,
+            description="RAM memory usage for LLMCommandGenerator",
+            unit=LLM_COMMAND_GENERATOR_CPU_MEMORY_USAGE_UNIT_NAME,
+        )
+
+        llm_command_generator_prompt_token_usage = meter.create_histogram(
+            name=LLM_COMMAND_GENERATOR_PROMPT_TOKEN_USAGE_METRIC_NAME,
+            description="LLMCommandGenerator prompt token length",
+            unit="1",
+        )
+
+        llm_response_duration_llm_command_generator = meter.create_histogram(
+            name=LLM_COMMAND_GENERATOR_LLM_RESPONSE_DURATION_METRIC_NAME,
+            description="The duration of LLMCommandGenerator's LLM call",
+            unit=DURATION_UNIT_NAME,
+        )
+
+        return {
+            LLM_COMMAND_GENERATOR_CPU_USAGE_METRIC_NAME: llm_command_generator_cpu_usage,  # noqa: E501
+            LLM_COMMAND_GENERATOR_MEMORY_USAGE_METRIC_NAME: llm_command_generator_memory_usage,  # noqa: E501
+            LLM_COMMAND_GENERATOR_PROMPT_TOKEN_USAGE_METRIC_NAME: llm_command_generator_prompt_token_usage,  # noqa: E501
+            LLM_COMMAND_GENERATOR_LLM_RESPONSE_DURATION_METRIC_NAME: llm_response_duration_llm_command_generator,  # noqa: E501
+        }
+
+    @staticmethod
+    def _create_llm_response_duration_instruments(meter: Meter) -> Dict[str, Any]:
+        llm_response_duration_enterprise_search = meter.create_histogram(
+            name=ENTERPRISE_SEARCH_POLICY_LLM_RESPONSE_DURATION_METRIC_NAME,
+            description="The duration of EnterpriseSearchPolicy's LLM call",
+            unit=DURATION_UNIT_NAME,
+        )
+
+        llm_response_duration_intentless = meter.create_histogram(
+            name=INTENTLESS_POLICY_LLM_RESPONSE_DURATION_METRIC_NAME,
+            description="The duration of IntentlessPolicy's LLM call",
+            unit=DURATION_UNIT_NAME,
+        )
+
+        llm_response_duration_contextual_nlg = meter.create_histogram(
+            name=CONTEXTUAL_RESPONSE_REPHRASER_LLM_RESPONSE_DURATION_METRIC_NAME,
+            description="The duration of ContextualResponseRephraser's LLM call",
+            unit=DURATION_UNIT_NAME,
+        )
+
+        return {
+            ENTERPRISE_SEARCH_POLICY_LLM_RESPONSE_DURATION_METRIC_NAME: llm_response_duration_enterprise_search,  # noqa: E501
+            INTENTLESS_POLICY_LLM_RESPONSE_DURATION_METRIC_NAME: llm_response_duration_intentless,  # noqa: E501
+            CONTEXTUAL_RESPONSE_REPHRASER_LLM_RESPONSE_DURATION_METRIC_NAME: llm_response_duration_contextual_nlg,  # noqa: E501
+        }
+
+    @staticmethod
+    def _create_client_request_instruments(
+        meter: Meter,
+    ) -> Dict[str, Any]:
+        client_request_duration = meter.create_histogram(
+            name=RASA_CLIENT_REQUEST_DURATION_METRIC_NAME,
+            description="The duration of the rasa client request",
+            unit=DURATION_UNIT_NAME,
+        )
+
+        client_request_body_size = meter.create_histogram(
+            name=RASA_CLIENT_REQUEST_BODY_SIZE_METRIC_NAME,
+            description="The rasa client request's body size",
+            unit="byte",
+        )
+
+        return {
+            RASA_CLIENT_REQUEST_DURATION_METRIC_NAME: client_request_duration,
+            RASA_CLIENT_REQUEST_BODY_SIZE_METRIC_NAME: client_request_body_size,
+        }

rasa/utils/beta.py

@@ -0,0 +1,83 @@
+import logging
+from textwrap import dedent
+
+from rasa.shared.exceptions import RasaException
+from rasa.utils.common import get_bool_env_variable
+
+logger = logging.getLogger(__name__)
+
+
+class BetaNotEnabledException(SystemExit, RasaException):
+    """Raised when a beta feature is not enabled."""
+
+    def __init__(self, feature_name: str, env_flag: str) -> None:
+        """Initializes the exception.
+
+        Args:
+            feature_name: The name of the feature.
+            env_flag: Name of the environment variable that enables the feature.
+        """
+        self.feature_name = feature_name
+        self.env_flag = env_flag
+        super().__init__(
+            dedent(
+                f"""
+            This release, including the {feature_name} feature, is a beta release
+            provided to you only for trial and evaluation. You cannot use this
+            beta software release as part of any production environment. This
+            beta software release is not intended for processing sensitive or
+            personal data. There may be bugs or errors - we provide it “as is”,
+            without any reps and warranties. We may never commercialize some beta
+            software. Your use of the {feature_name} feature and any other
+            software or material included in this beta release is subject to
+            the Beta Software Terms (https://rasa.com/beta-terms/).
+            Please make sure you read these terms and conditions prior to
+            using this beta release.
+
+            You need to explicitly enable the {feature_name} feature, before
+            usage. Set the `{env_flag}=true` environment variable before
+            running the command again.
+            """
+            )
+        )
+
+
+def ensure_beta_feature_is_enabled(feature_name: str, env_flag: str) -> None:
+    """Checks if a beta feature is enabled. Raises an exception if not.
+
+    If the feature is enabled, a usage warning is printed.
+
+    Args:
+        feature_name: The name of the feature.
+        env_flag: The name of the environment variable that enables the feature.
+    """
+    if not get_bool_env_variable(env_flag, default_variable_value=False):
+        raise BetaNotEnabledException(feature_name, env_flag)
+
+    print_beta_usage_info(feature_name, env_flag)
+
+
+def print_beta_usage_info(feature_name: str, env_flag: str) -> None:
+    """Prints a usage warning about a beta feature.
+
+    Args:
+        feature_name: The name of the feature.
+        env_flag: The name of the environment variable that enables the feature.
+    """
+    logger.warning(
+        dedent(
+            f"""\
+        The {feature_name} feature is currently released in a beta version.
+
+        The feature might change in the future 🔬. Your download and use of
+        this beta release are subject to the Beta Software Terms
+        (https://rasa.com/beta-terms/).
+        It's not intended for production. Don't use it to process sensitive data.
+        If you do, it's at your own risk. It's not mandatory to download
+        or use this software release. We're looking forward to your feedback.
+
+        If you want to disable this beta feature, set the environment variable
+        `{env_flag}=false`.
+    """
+        )
+    )

rasa/utils/cli.py

@@ -0,0 +1,27 @@
+import argparse
+from typing import Text
+
+
+def remove_argument_from_parser(
+    parser: argparse.ArgumentParser, argument: Text
+) -> None:
+    """Remove argument from parse.
+
+    A public API for this use case doesn't exist; so we use a private one.
+    We should be able to detect breaking changes of this API through unit-testing.
+
+    Inspired by this Stackoverflow answer: https://stackoverflow.com/a/32809642
+
+    Args:
+        parser: an argument parser instance
+        argument: the argument to remove, e.g. -f or --foo
+    """
+    for action in parser._actions:
+        for option in action.option_strings:
+            if option == argument:
+                parser._handle_conflict_resolve(
+                    None, [(option, action)]  # type: ignore[arg-type]
+                )
+                return
+
+    raise LookupError(f"Cannot find argument {argument}")