rasa-pro 3.9.18__py3-none-any.whl → 3.10.16__py3-none-any.whl

rasa/core/channels/voice_aware/jambonz_protocol.py

@@ -0,0 +1,344 @@
+from dataclasses import dataclass, field
+import json
+import uuid
+from typing import Any, Awaitable, Callable, Dict, List, Text
+
+import structlog
+from rasa.core.channels.channel import UserMessage
+from sanic import Websocket  # type: ignore[attr-defined]
+
+
+structlogger = structlog.get_logger()
+
+
+@dataclass
+class NewSessionMessage:
+    """Message indicating a new session has been started."""
+
+    call_sid: str
+    message_id: str
+
+    @staticmethod
+    def from_message(message: Dict[str, Any]) -> "NewSessionMessage":
+        return NewSessionMessage(
+            message.get("call_sid"),
+            message.get("msgid"),
+        )
+
+
+@dataclass
+class Transcript:
+    """Transcript of a spoken utterance."""
+
+    text: str
+    confidence: float
+
+
+@dataclass
+class TranscriptResult:
+    """Result of an ASR call with potential transcripts."""
+
+    call_sid: str
+    message_id: str
+    is_final: bool
+    transcripts: List[Transcript] = field(default_factory=list)
+
+    @staticmethod
+    def from_speech_result(message: Dict[str, Any]) -> "TranscriptResult":
+        return TranscriptResult(
+            message.get("call_sid"),
+            message.get("msgid"),
+            message.get("data", {}).get("speech", {}).get("is_final", True),
+            transcripts=[
+                Transcript(t.get("transcript", ""), t.get("confidence", 1.0))
+                for t in message.get("data", {})
+                .get("speech", {})
+                .get("alternatives", [])
+            ],
+        )
+
+    @staticmethod
+    def from_dtmf_result(message: Dict[str, Any]) -> "TranscriptResult":
+        """Create a transcript result from a DTMF result.
+
+        We use the dtmf as the text with confidence 1.0
+        """
+        return TranscriptResult(
+            message.get("call_sid"),
+            message.get("msgid"),
+            is_final=True,
+            transcripts=[
+                Transcript(str(message.get("data", {}).get("digits", "")), 1.0)
+            ],
+        )
+
+
+@dataclass
+class CallStatusChanged:
+    """Message indicating a change in the call status."""
+
+    call_sid: str
+    status: str
+
+    @staticmethod
+    def from_message(message: Dict[str, Any]) -> "CallStatusChanged":
+        return CallStatusChanged(
+            message.get("call_sid"), message.get("data", {}).get("call_status")
+        )
+
+
+@dataclass
+class SessionReconnect:
+    """Message indicating a session has reconnected."""
+
+    call_sid: str
+
+    @staticmethod
+    def from_message(message: Dict[str, Any]) -> "SessionReconnect":
+        return SessionReconnect(message.get("call_sid"))
+
+
+@dataclass
+class VerbStatusChanged:
+    """Message indicating a change in the status of a verb."""
+
+    call_sid: str
+    event: str
+    id: str
+    name: str
+
+    @staticmethod
+    def from_message(message: Dict[str, Any]) -> "VerbStatusChanged":
+        return VerbStatusChanged(
+            message.get("call_sid"),
+            message.get("data", {}).get("event"),
+            message.get("data", {}).get("id"),
+            message.get("data", {}).get("name"),
+        )
+
+
+@dataclass
+class GatherTimeout:
+    """Message indicating a gather timeout."""
+
+    call_sid: str
+
+    @staticmethod
+    def from_message(message: Dict[str, Any]) -> "GatherTimeout":
+        return GatherTimeout(message.get("call_sid"))
+
+
+async def websocket_message_handler(
+    message_dump: str,
+    on_new_message: Callable[[UserMessage], Awaitable[Any]],
+    ws: Websocket,
+) -> None:
+    """Handle incoming messages from the websocket."""
+    message = json.loads(message_dump)
+
+    # parse and handle the different message types
+    if message.get("type") == "session:new":
+        new_session = NewSessionMessage.from_message(message)
+        await handle_new_session(new_session, on_new_message, ws)
+    elif message.get("type") == "session:reconnect":
+        session_reconnect = SessionReconnect.from_message(message)
+        await handle_session_reconnect(session_reconnect)
+    elif message.get("type") == "call:status":
+        call_status = CallStatusChanged.from_message(message)
+        await handle_call_status(call_status)
+    elif message.get("type") == "verb:hook" and message.get("hook") == "/gather":
+        hook_trigger_reason = message.get("data", {}).get("reason")
+
+        if hook_trigger_reason == "speechDetected":
+            transcript = TranscriptResult.from_speech_result(message)
+            await handle_gather_completed(transcript, on_new_message, ws)
+        elif hook_trigger_reason == "timeout":
+            gather_timeout = GatherTimeout.from_message(message)
+            await handle_gather_timeout(gather_timeout, ws)
+        elif hook_trigger_reason == "dtmfDetected":
+            # for now, let's handle it as normal user input with a
+            # confidence of 1.0
+            transcript = TranscriptResult.from_dtmf_result(message)
+            await handle_gather_completed(transcript, on_new_message, ws)
+        else:
+            structlogger.debug(
+                "jambonz.websocket.message.verb_hook",
+                call_sid=message.get("call_sid"),
+                reason=hook_trigger_reason,
+                message=message,
+            )
+    elif message.get("type") == "verb:status":
+        verb_status = VerbStatusChanged.from_message(message)
+        await handle_verb_status(verb_status)
+    elif message.get("type") == "jambonz:error":
+        # jambonz ran into a fatal error handling the call. the call will be
+        # terminated.
+        structlogger.error("jambonz.websocket.message.error", message=message)
+    else:
+        structlogger.warning("jambonz.websocket.message.unknown_type", message=message)
+
+
+async def handle_new_session(
+    message: NewSessionMessage,
+    on_new_message: Callable[[UserMessage], Awaitable[Any]],
+    ws: Websocket,
+) -> None:
+    """Handle new session message."""
+    from rasa.core.channels.voice_aware.jambonz import JambonzWebsocketOutput
+
+    structlogger.debug("jambonz.websocket.message.new_call", call_sid=message.call_sid)
+    output_channel = JambonzWebsocketOutput(ws, message.call_sid)
+    user_msg = UserMessage(
+        text="/session_start",
+        output_channel=output_channel,
+        sender_id=message.call_sid,
+        metadata={},
+    )
+    await send_config_ack(message.message_id, ws)
+    await on_new_message(user_msg)
+    await send_gather_input(ws)
+
+
+async def handle_gather_completed(
+    transcript_result: TranscriptResult,
+    on_new_message: Callable[[UserMessage], Awaitable[Any]],
+    ws: Websocket,
+) -> None:
+    """Handle changes to commands we have send to jambonz.
+
+    This includes results of gather calles with their transcription.
+    """
+    from rasa.core.channels.voice_aware.jambonz import JambonzWebsocketOutput
+
+    if not transcript_result.is_final:
+        # in case of a non final transcript, we are going to wait for the final
+        # one and ignore the partial one
+        structlogger.debug(
+            "jambonz.websocket.message.transcript_partial",
+            call_sid=transcript_result.call_sid,
+            number_of_transcripts=len(transcript_result.transcripts),
+        )
+        return
+
+    if transcript_result.transcripts:
+        most_likely_transcript = transcript_result.transcripts[0]
+        output_channel = JambonzWebsocketOutput(ws, transcript_result.call_sid)
+        user_msg = UserMessage(
+            text=most_likely_transcript.text,
+            output_channel=output_channel,
+            sender_id=transcript_result.call_sid,
+            metadata={},
+        )
+        structlogger.debug(
+            "jambonz.websocket.message.transcript",
+            call_sid=transcript_result.call_sid,
+            transcript=most_likely_transcript.text,
+            confidence=most_likely_transcript.confidence,
+            number_of_transcripts=len(transcript_result.transcripts),
+        )
+        await on_new_message(user_msg)
+    else:
+        structlogger.warning(
+            "jambonz.websocket.message.no_transcript",
+            call_sid=transcript_result.call_sid,
+        )
+    await send_gather_input(ws)
+
+
+async def handle_gather_timeout(gather_timeout: GatherTimeout, ws: Websocket) -> None:
+    """Handle gather timeout."""
+    structlogger.debug(
+        "jambonz.websocket.message.gather_timeout",
+        call_sid=gather_timeout.call_sid,
+    )
+    # TODO: figure out how to handle timeouts
+    await send_ws_text_message(ws, "I'm sorry, I didn't catch that.")
+    await send_gather_input(ws)
+
+
+async def handle_call_status(call_status: CallStatusChanged) -> None:
+    """Handle changes in the call status."""
+    structlogger.debug(
+        "jambonz.websocket.message.call_status_changed",
+        call_sid=call_status.call_sid,
+        message=call_status.status,
+    )
+
+
+async def handle_session_reconnect(session_reconnect: SessionReconnect) -> None:
+    """Handle session reconnect message."""
+    # there is nothing we need to do atm when a session reconnects.
+    # this happens if jambonz looses the websocket connection and reconnects
+    structlogger.debug(
+        "jambonz.websocket.message.session_reconnect",
+        call_sid=session_reconnect.call_sid,
+    )
+
+
+async def handle_verb_status(verb_status: VerbStatusChanged) -> None:
+    """Handle changes in the status of a verb."""
+    structlogger.debug(
+        "jambonz.websocket.message.verb_status_changed",
+        call_sid=verb_status.call_sid,
+        event_type=verb_status.event,
+        id=verb_status.id,
+        name=verb_status.name,
+    )
+
+
+async def send_config_ack(message_id: str, ws: Websocket) -> None:
+    """Send an ack message to jambonz including the configuration."""
+    await ws.send(
+        json.dumps(
+            {
+                "type": "ack",
+                "msgid": message_id,
+                "data": [{"config": {"notifyEvents": True}}],
+            }
+        )
+    )
+
+
+async def send_gather_input(ws: Websocket) -> None:
+    """Send a gather input command to jambonz."""
+    await ws.send(
+        json.dumps(
+            {
+                "type": "command",
+                "command": "redirect",
+                "queueCommand": True,
+                "data": [
+                    {
+                        "gather": {
+                            "input": ["speech", "digits"],
+                            "minDigits": 1,
+                            "id": uuid.uuid4().hex,
+                            "actionHook": "/gather",
+                        }
+                    }
+                ],
+            }
+        )
+    )
+
+
+async def send_ws_text_message(ws: Websocket, text: Text) -> None:
+    """Send a text message to the websocket using the jambonz interface."""
+    await ws.send(
+        json.dumps(
+            {
+                "type": "command",
+                "command": "redirect",
+                "queueCommand": True,
+                "data": [
+                    {
+                        "say": {
+                            # id can be used for status notifications
+                            "id": uuid.uuid4().hex,
+                            "text": text,
+                        }
+                    }
+                ],
+            }
+        )
+    )

rasa/core/channels/voice_aware/utils.py

@@ -0,0 +1,20 @@
+import structlog
+
+from rasa.utils.licensing import (
+    PRODUCT_AREA,
+    VOICE_SCOPE,
+    validate_license_from_env,
+)
+
+structlogger = structlog.get_logger()
+
+
+def validate_voice_license_scope() -> None:
+    """Validate that the correct license scope is present."""
+    structlogger.info(
+        f"Validating current Rasa Pro license scope which must include "
+        f"the '{VOICE_SCOPE}' scope to use the voice channel."
+    )
+
+    voice_product_scope = PRODUCT_AREA + " " + VOICE_SCOPE
+    validate_license_from_env(product_area=voice_product_scope)

rasa/core/constants.py

@@ -61,7 +61,6 @@ SEARCH_POLICY_PRIORITY = CHAT_POLICY_PRIORITY + 1
 # flow policy priority
 FLOW_POLICY_PRIORITY = SEARCH_POLICY_PRIORITY + 1
 
-
 DIALOGUE = "dialogue"
 
 # RabbitMQ message property header added to events published using `rasa export`
@@ -105,3 +104,9 @@ DEFAULT_TEMPLATE_ENGINE = RASA_FORMAT_TEMPLATE_ENGINE
 # configuration parameter used to specify the template engine to use
 # for a response
 TEMPLATE_ENGINE_CONFIG_KEY = "template"
+
+# metadata keys for bot utterance events
+UTTER_SOURCE_METADATA_KEY = "utter_source"
+DOMAIN_GROUND_TRUTH_METADATA_KEY = "domain_ground_truth"
+ACTIVE_FLOW_METADATA_KEY = "active_flow"
+STEP_ID_METADATA_KEY = "step_id"

rasa/core/information_retrieval/faiss.py

@@ -2,9 +2,10 @@ from pathlib import Path
 from typing import TYPE_CHECKING, List, Optional, Text, Any, Dict
 
 import structlog
-from langchain.document_loaders import DirectoryLoader, TextLoader
 from langchain.text_splitter import RecursiveCharacterTextSplitter
-from langchain.vectorstores.faiss import FAISS
+from langchain_community.document_loaders.text import TextLoader
+from langchain_community.document_loaders.directory import DirectoryLoader
+from langchain_community.vectorstores.faiss import FAISS
 from rasa.utils.endpoints import EndpointConfig
 
 from rasa.core.information_retrieval import (
@@ -46,7 +47,9 @@ class FAISS_Store(InformationRetrieval):
             logger.info(
                 "information_retrieval.faiss_store.load_index", path=path.absolute()
             )
-            self.index = FAISS.load_local(str(path), embeddings)
+            self.index = FAISS.load_local(
+                str(path), embeddings, allow_dangerous_deserialization=True
+            )
 
     @staticmethod
     def load_documents(docs_folder: str) -> List["Document"]:
@@ -114,7 +117,7 @@ class FAISS_Store(InformationRetrieval):
     ) -> SearchResultList:
         logger.debug("information_retrieval.faiss_store.search", query=query)
         try:
-            documents = await self.index.as_retriever().aget_relevant_documents(query)
+            documents = await self.index.as_retriever().ainvoke(query)
         except Exception as exc:
             raise InformationRetrievalException from exc
 

rasa/core/information_retrieval/information_retrieval.py

@@ -19,6 +19,14 @@ logger = structlog.get_logger()
 
 @dataclass
 class SearchResult:
+    """A search result object.
+
+    Attributes:
+        text: The text content of the retrieved document result.
+        metadata: The metadata associated with the document result.
+        score: The score of the search result.
+    """
+
     text: str
     metadata: dict
     score: Optional[float] = None

rasa/core/information_retrieval/milvus.py

@@ -1,7 +1,7 @@
 from typing import Text, Any, Dict
 
 import structlog
-from langchain.vectorstores.milvus import Milvus
+from langchain_community.vectorstores.milvus import Milvus
 from rasa.utils.endpoints import EndpointConfig
 
 from rasa.core.information_retrieval import (
@@ -48,5 +48,12 @@ class Milvus_Store(InformationRetrieval):
         except Exception as exc:
             raise InformationRetrievalException from exc
 
-        filtered_hits = [doc for doc, score in hits if score >= threshold]
+        scores = [score for _, score in hits]
+        logger.debug(
+            "information_retrieval.milvus_store.search_results_before_threshold",
+            scores=scores,
+        )
+        # Milvus uses Euclidean distance metric by default
+        # so the lower the score, the better the match.
+        filtered_hits = [doc for doc, score in hits if score <= threshold]
         return SearchResultList.from_document_list(filtered_hits)

rasa/core/information_retrieval/qdrant.py

@@ -1,7 +1,7 @@
 from typing import Text, Any, Dict
 
 import structlog
-from langchain.vectorstores.qdrant import Qdrant
+from langchain_community.vectorstores.qdrant import Qdrant
 from pydantic import ValidationError
 from qdrant_client import QdrantClient
 from rasa.utils.endpoints import EndpointConfig

rasa/core/nlg/contextual_response_rephraser.py

@@ -5,6 +5,15 @@ from jinja2 import Template
 
 from rasa import telemetry
 from rasa.core.nlg.response import TemplatedNaturalLanguageGenerator
+from rasa.shared.constants import (
+    LLM_CONFIG_KEY,
+    MODEL_CONFIG_KEY,
+    MODEL_NAME_CONFIG_KEY,
+    PROMPT_CONFIG_KEY,
+    PROVIDER_CONFIG_KEY,
+    OPENAI_PROVIDER,
+    TIMEOUT_CONFIG_KEY,
+)
 from rasa.shared.core.domain import KEY_RESPONSES_TEXT, Domain
 from rasa.shared.core.events import BotUttered, UserUttered
 from rasa.shared.core.trackers import DialogueStateTracker
@@ -15,6 +24,7 @@ from rasa.shared.utils.llm import (
     combine_custom_and_default_config,
     get_prompt_template,
     llm_factory,
+    try_instantiate_llm_client,
 )
 from rasa.utils.endpoints import EndpointConfig
 
@@ -31,11 +41,11 @@ RESPONSE_REPHRASING_TEMPLATE_KEY = "rephrase_prompt"
 DEFAULT_REPHRASE_ALL = False
 
 DEFAULT_LLM_CONFIG = {
-    "_type": "openai",
-    "request_timeout": 5,
+    PROVIDER_CONFIG_KEY: OPENAI_PROVIDER,
+    MODEL_CONFIG_KEY: DEFAULT_OPENAI_GENERATE_MODEL_NAME,
     "temperature": 0.3,
-    "model_name": DEFAULT_OPENAI_GENERATE_MODEL_NAME,
     "max_tokens": DEFAULT_OPENAI_MAX_GENERATED_TOKENS,
+    TIMEOUT_CONFIG_KEY: 5,
 }
 
 DEFAULT_RESPONSE_VARIATION_PROMPT_TEMPLATE = """The following is a conversation with
@@ -78,7 +88,7 @@ class ContextualResponseRephraser(TemplatedNaturalLanguageGenerator):
 
         self.nlg_endpoint = endpoint_config
         self.prompt_template = get_prompt_template(
-            self.nlg_endpoint.kwargs.get("prompt"),
+            self.nlg_endpoint.kwargs.get(PROMPT_CONFIG_KEY),
             DEFAULT_RESPONSE_VARIATION_PROMPT_TEMPLATE,
         )
         self.rephrase_all = self.nlg_endpoint.kwargs.get(
@@ -87,6 +97,12 @@ class ContextualResponseRephraser(TemplatedNaturalLanguageGenerator):
         self.trace_prompt_tokens = self.nlg_endpoint.kwargs.get(
             "trace_prompt_tokens", False
         )
+        try_instantiate_llm_client(
+            self.nlg_endpoint.kwargs.get(LLM_CONFIG_KEY),
+            DEFAULT_LLM_CONFIG,
+            "contextual_response_rephraser.init",
+            "ContextualResponseRephraser",
+        )
 
     def _last_message_if_human(self, tracker: DialogueStateTracker) -> Optional[str]:
         """Returns the latest message from the tracker.
@@ -115,10 +131,13 @@ class ContextualResponseRephraser(TemplatedNaturalLanguageGenerator):
         Returns:
             generated text
         """
-        llm = llm_factory(self.nlg_endpoint.kwargs.get("llm"), DEFAULT_LLM_CONFIG)
+        llm = llm_factory(
+            self.nlg_endpoint.kwargs.get(LLM_CONFIG_KEY), DEFAULT_LLM_CONFIG
+        )
 
         try:
-            return await llm.apredict(prompt)
+            llm_response = await llm.acompletion(prompt)
+            return llm_response.choices[0]
         except Exception as e:
             # unfortunately, langchain does not wrap LLM exceptions which means
             # we have to catch all exceptions here
@@ -128,7 +147,7 @@ class ContextualResponseRephraser(TemplatedNaturalLanguageGenerator):
     def llm_property(self, prop: str) -> Optional[str]:
         """Returns a property of the LLM provider."""
         return combine_custom_and_default_config(
-            self.nlg_endpoint.kwargs.get("llm"), DEFAULT_LLM_CONFIG
+            self.nlg_endpoint.kwargs.get(LLM_CONFIG_KEY), DEFAULT_LLM_CONFIG
         ).get(prop)
 
     def custom_prompt_template(self, prompt_template: str) -> Optional[str]:
@@ -161,7 +180,9 @@ class ContextualResponseRephraser(TemplatedNaturalLanguageGenerator):
         Returns:
         The history for the prompt.
         """
-        llm = llm_factory(self.nlg_endpoint.kwargs.get("llm"), DEFAULT_LLM_CONFIG)
+        llm = llm_factory(
+            self.nlg_endpoint.kwargs.get(LLM_CONFIG_KEY), DEFAULT_LLM_CONFIG
+        )
         return await summarize_conversation(tracker, llm, max_turns=5)
 
     async def rephrase(
@@ -202,8 +223,9 @@ class ContextualResponseRephraser(TemplatedNaturalLanguageGenerator):
         telemetry.track_response_rephrase(
             rephrase_all=self.rephrase_all,
             custom_prompt_template=self.custom_prompt_template(prompt_template_text),
-            llm_type=self.llm_property("_type"),
-            llm_model=self.llm_property("model") or self.llm_property("model_name"),
+            llm_type=self.llm_property(PROVIDER_CONFIG_KEY),
+            llm_model=self.llm_property(MODEL_CONFIG_KEY)
+            or self.llm_property(MODEL_NAME_CONFIG_KEY),
         )
         if not (updated_text := await self._generate_llm_response(prompt)):
             # If the LLM fails to generate a response, we

rasa/core/nlg/summarize.py

@@ -2,8 +2,8 @@ from typing import Optional
 
 import structlog
 from jinja2 import Template
-from langchain.llms.base import BaseLLM
 from rasa.core.tracker_store import DialogueStateTracker
+from rasa.shared.providers.llm.llm_client import LLMClient
 from rasa.shared.utils.llm import (
     tracker_as_readable_transcript,
 )
@@ -43,7 +43,7 @@ def _create_summarization_prompt(
 
 async def summarize_conversation(
     tracker: DialogueStateTracker,
-    llm: BaseLLM,
+    llm: LLMClient,
     max_turns: Optional[int] = MAX_TURNS_DEFAULT,
 ) -> str:
     """Summarizes the dialogue using the LLM.
@@ -58,7 +58,8 @@ async def summarize_conversation(
     """
     prompt = _create_summarization_prompt(tracker, max_turns)
     try:
-        summarization = (await llm.apredict(prompt)).strip()
+        llm_response = await llm.acompletion(prompt)
+        summarization = llm_response.choices[0].strip()
         structlogger.debug(
             "summarization.success", summarization=summarization, prompt=prompt
         )