rasa-pro 3.13.7__py3-none-any.whl → 3.14.0.dev1__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.

Potentially problematic release.


This version of rasa-pro might be problematic. Click here for more details.

Files changed (178) hide show
  1. rasa/agents/__init__.py +0 -0
  2. rasa/agents/agent_factory.py +122 -0
  3. rasa/agents/agent_manager.py +162 -0
  4. rasa/agents/constants.py +31 -0
  5. rasa/agents/core/__init__.py +0 -0
  6. rasa/agents/core/agent_protocol.py +108 -0
  7. rasa/agents/core/types.py +70 -0
  8. rasa/agents/exceptions.py +8 -0
  9. rasa/agents/protocol/__init__.py +5 -0
  10. rasa/agents/protocol/a2a/__init__.py +0 -0
  11. rasa/agents/protocol/a2a/a2a_agent.py +51 -0
  12. rasa/agents/protocol/mcp/__init__.py +0 -0
  13. rasa/agents/protocol/mcp/mcp_base_agent.py +697 -0
  14. rasa/agents/protocol/mcp/mcp_open_agent.py +275 -0
  15. rasa/agents/protocol/mcp/mcp_task_agent.py +447 -0
  16. rasa/agents/schemas/__init__.py +6 -0
  17. rasa/agents/schemas/agent_input.py +24 -0
  18. rasa/agents/schemas/agent_output.py +26 -0
  19. rasa/agents/schemas/agent_tool_result.py +51 -0
  20. rasa/agents/schemas/agent_tool_schema.py +112 -0
  21. rasa/agents/templates/__init__.py +0 -0
  22. rasa/agents/templates/mcp_open_agent_prompt_template.jinja2 +15 -0
  23. rasa/agents/templates/mcp_task_agent_prompt_template.jinja2 +13 -0
  24. rasa/agents/utils.py +72 -0
  25. rasa/api.py +5 -0
  26. rasa/cli/arguments/default_arguments.py +12 -0
  27. rasa/cli/arguments/run.py +2 -0
  28. rasa/cli/dialogue_understanding_test.py +4 -0
  29. rasa/cli/e2e_test.py +4 -0
  30. rasa/cli/inspect.py +3 -0
  31. rasa/cli/llm_fine_tuning.py +5 -0
  32. rasa/cli/run.py +4 -0
  33. rasa/cli/shell.py +3 -0
  34. rasa/cli/train.py +2 -2
  35. rasa/constants.py +6 -0
  36. rasa/core/actions/action.py +69 -39
  37. rasa/core/actions/action_run_slot_rejections.py +1 -1
  38. rasa/core/agent.py +16 -0
  39. rasa/core/available_agents.py +196 -0
  40. rasa/core/available_endpoints.py +30 -0
  41. rasa/core/channels/development_inspector.py +47 -14
  42. rasa/core/channels/inspector/dist/assets/{arc-0b11fe30.js → arc-2e78c586.js} +1 -1
  43. rasa/core/channels/inspector/dist/assets/{blockDiagram-38ab4fdb-9eef30a7.js → blockDiagram-38ab4fdb-806b712e.js} +1 -1
  44. rasa/core/channels/inspector/dist/assets/{c4Diagram-3d4e48cf-03e94f28.js → c4Diagram-3d4e48cf-0745efa9.js} +1 -1
  45. rasa/core/channels/inspector/dist/assets/channel-c436ca7c.js +1 -0
  46. rasa/core/channels/inspector/dist/assets/{classDiagram-70f12bd4-95c09eba.js → classDiagram-70f12bd4-7bd1082b.js} +1 -1
  47. rasa/core/channels/inspector/dist/assets/{classDiagram-v2-f2320105-38e8446c.js → classDiagram-v2-f2320105-d937ba49.js} +1 -1
  48. rasa/core/channels/inspector/dist/assets/clone-50dd656b.js +1 -0
  49. rasa/core/channels/inspector/dist/assets/{createText-2e5e7dd3-57dc3038.js → createText-2e5e7dd3-a2a564ca.js} +1 -1
  50. rasa/core/channels/inspector/dist/assets/{edges-e0da2a9e-4bac0545.js → edges-e0da2a9e-b5256940.js} +1 -1
  51. rasa/core/channels/inspector/dist/assets/{erDiagram-9861fffd-81795c90.js → erDiagram-9861fffd-e6883ad2.js} +1 -1
  52. rasa/core/channels/inspector/dist/assets/{flowDb-956e92f1-89489ae6.js → flowDb-956e92f1-e576fc02.js} +1 -1
  53. rasa/core/channels/inspector/dist/assets/{flowDiagram-66a62f08-cd152627.js → flowDiagram-66a62f08-2e298d01.js} +1 -1
  54. rasa/core/channels/inspector/dist/assets/flowDiagram-v2-96b9c2cf-2b2aeaf8.js +1 -0
  55. rasa/core/channels/inspector/dist/assets/{flowchart-elk-definition-4a651766-3da369bc.js → flowchart-elk-definition-4a651766-dd7b150a.js} +1 -1
  56. rasa/core/channels/inspector/dist/assets/{ganttDiagram-c361ad54-85ec16f8.js → ganttDiagram-c361ad54-5b79575c.js} +1 -1
  57. rasa/core/channels/inspector/dist/assets/{gitGraphDiagram-72cf32ee-495bc140.js → gitGraphDiagram-72cf32ee-3016f40a.js} +1 -1
  58. rasa/core/channels/inspector/dist/assets/{graph-1ec4d266.js → graph-3e19170f.js} +1 -1
  59. rasa/core/channels/inspector/dist/assets/index-1bd9135e.js +1353 -0
  60. rasa/core/channels/inspector/dist/assets/{index-3862675e-0a0e97c9.js → index-3862675e-eb9c86de.js} +1 -1
  61. rasa/core/channels/inspector/dist/assets/{infoDiagram-f8f76790-4d54bcde.js → infoDiagram-f8f76790-b4280e4d.js} +1 -1
  62. rasa/core/channels/inspector/dist/assets/{journeyDiagram-49397b02-dc097114.js → journeyDiagram-49397b02-556091f8.js} +1 -1
  63. rasa/core/channels/inspector/dist/assets/{layout-1a08981e.js → layout-08436411.js} +1 -1
  64. rasa/core/channels/inspector/dist/assets/{line-95f7f1d3.js → line-683c4f3b.js} +1 -1
  65. rasa/core/channels/inspector/dist/assets/{linear-97e69543.js → linear-cee6d791.js} +1 -1
  66. rasa/core/channels/inspector/dist/assets/{mindmap-definition-fc14e90a-8c71ff03.js → mindmap-definition-fc14e90a-a0bf0b1a.js} +1 -1
  67. rasa/core/channels/inspector/dist/assets/{pieDiagram-8a3498a8-f14c71c7.js → pieDiagram-8a3498a8-3730d5c4.js} +1 -1
  68. rasa/core/channels/inspector/dist/assets/{quadrantDiagram-120e2f19-f1d3c9ff.js → quadrantDiagram-120e2f19-12a20fed.js} +1 -1
  69. rasa/core/channels/inspector/dist/assets/{requirementDiagram-deff3bca-bfa2412f.js → requirementDiagram-deff3bca-b9732102.js} +1 -1
  70. rasa/core/channels/inspector/dist/assets/{sankeyDiagram-04a897e0-53f2c97b.js → sankeyDiagram-04a897e0-a2e72776.js} +1 -1
  71. rasa/core/channels/inspector/dist/assets/{sequenceDiagram-704730f1-319d7c0e.js → sequenceDiagram-704730f1-8b7a76bb.js} +1 -1
  72. rasa/core/channels/inspector/dist/assets/{stateDiagram-587899a1-76a09418.js → stateDiagram-587899a1-e65853ac.js} +1 -1
  73. rasa/core/channels/inspector/dist/assets/{stateDiagram-v2-d93cdb3a-a67f15d4.js → stateDiagram-v2-d93cdb3a-6f58a44b.js} +1 -1
  74. rasa/core/channels/inspector/dist/assets/{styles-6aaf32cf-0654e7c3.js → styles-6aaf32cf-df25b934.js} +1 -1
  75. rasa/core/channels/inspector/dist/assets/{styles-9a916d00-1394bb9d.js → styles-9a916d00-88357141.js} +1 -1
  76. rasa/core/channels/inspector/dist/assets/{styles-c10674c1-e4c5bdae.js → styles-c10674c1-d600174d.js} +1 -1
  77. rasa/core/channels/inspector/dist/assets/{svgDrawCommon-08f97a94-50957104.js → svgDrawCommon-08f97a94-4adc3e0b.js} +1 -1
  78. rasa/core/channels/inspector/dist/assets/{timeline-definition-85554ec2-b0885a6a.js → timeline-definition-85554ec2-42816fa1.js} +1 -1
  79. rasa/core/channels/inspector/dist/assets/{xychartDiagram-e933f94c-79e6541a.js → xychartDiagram-e933f94c-621eb66a.js} +1 -1
  80. rasa/core/channels/inspector/dist/index.html +2 -2
  81. rasa/core/channels/inspector/index.html +1 -1
  82. rasa/core/channels/inspector/src/App.tsx +53 -7
  83. rasa/core/channels/inspector/src/components/Chat.tsx +3 -2
  84. rasa/core/channels/inspector/src/components/DiagramFlow.tsx +1 -1
  85. rasa/core/channels/inspector/src/components/DialogueStack.tsx +7 -5
  86. rasa/core/channels/inspector/src/components/LatencyDisplay.tsx +268 -0
  87. rasa/core/channels/inspector/src/components/LoadingSpinner.tsx +6 -2
  88. rasa/core/channels/inspector/src/helpers/audio/audiostream.ts +8 -3
  89. rasa/core/channels/inspector/src/helpers/formatters.ts +24 -3
  90. rasa/core/channels/inspector/src/theme/base/styles.ts +19 -1
  91. rasa/core/channels/inspector/src/types.ts +12 -0
  92. rasa/core/channels/studio_chat.py +125 -34
  93. rasa/core/channels/voice_ready/twilio_voice.py +1 -1
  94. rasa/core/channels/voice_stream/audiocodes.py +9 -6
  95. rasa/core/channels/voice_stream/browser_audio.py +39 -4
  96. rasa/core/channels/voice_stream/call_state.py +13 -2
  97. rasa/core/channels/voice_stream/genesys.py +16 -13
  98. rasa/core/channels/voice_stream/jambonz.py +13 -11
  99. rasa/core/channels/voice_stream/twilio_media_streams.py +14 -13
  100. rasa/core/channels/voice_stream/util.py +11 -1
  101. rasa/core/channels/voice_stream/voice_channel.py +101 -29
  102. rasa/core/constants.py +4 -0
  103. rasa/core/nlg/contextual_response_rephraser.py +11 -7
  104. rasa/core/nlg/generator.py +21 -5
  105. rasa/core/nlg/response.py +43 -6
  106. rasa/core/nlg/translate.py +8 -0
  107. rasa/core/policies/enterprise_search_policy.py +4 -2
  108. rasa/core/policies/flow_policy.py +2 -2
  109. rasa/core/policies/flows/flow_executor.py +374 -35
  110. rasa/core/policies/flows/mcp_tool_executor.py +240 -0
  111. rasa/core/processor.py +6 -1
  112. rasa/core/run.py +8 -1
  113. rasa/core/utils.py +21 -1
  114. rasa/dialogue_understanding/commands/__init__.py +8 -0
  115. rasa/dialogue_understanding/commands/cancel_flow_command.py +97 -4
  116. rasa/dialogue_understanding/commands/chit_chat_answer_command.py +11 -0
  117. rasa/dialogue_understanding/commands/continue_agent_command.py +91 -0
  118. rasa/dialogue_understanding/commands/knowledge_answer_command.py +11 -0
  119. rasa/dialogue_understanding/commands/restart_agent_command.py +146 -0
  120. rasa/dialogue_understanding/commands/start_flow_command.py +129 -8
  121. rasa/dialogue_understanding/commands/utils.py +6 -2
  122. rasa/dialogue_understanding/generator/command_parser.py +4 -0
  123. rasa/dialogue_understanding/generator/llm_based_command_generator.py +50 -12
  124. rasa/dialogue_understanding/generator/prompt_templates/agent_command_prompt_v2_claude_3_5_sonnet_20240620_template.jinja2 +61 -0
  125. rasa/dialogue_understanding/generator/prompt_templates/agent_command_prompt_v2_gpt_4o_2024_11_20_template.jinja2 +61 -0
  126. rasa/dialogue_understanding/generator/prompt_templates/agent_command_prompt_v3_claude_3_5_sonnet_20240620_template.jinja2 +81 -0
  127. rasa/dialogue_understanding/generator/prompt_templates/agent_command_prompt_v3_gpt_4o_2024_11_20_template.jinja2 +81 -0
  128. rasa/dialogue_understanding/generator/single_step/compact_llm_command_generator.py +7 -6
  129. rasa/dialogue_understanding/generator/single_step/search_ready_llm_command_generator.py +7 -6
  130. rasa/dialogue_understanding/generator/single_step/single_step_based_llm_command_generator.py +41 -2
  131. rasa/dialogue_understanding/patterns/continue_interrupted.py +163 -1
  132. rasa/dialogue_understanding/patterns/default_flows_for_patterns.yml +51 -7
  133. rasa/dialogue_understanding/stack/dialogue_stack.py +123 -2
  134. rasa/dialogue_understanding/stack/frames/flow_stack_frame.py +57 -0
  135. rasa/dialogue_understanding/stack/utils.py +3 -2
  136. rasa/dialogue_understanding_test/du_test_runner.py +7 -2
  137. rasa/dialogue_understanding_test/du_test_schema.yml +3 -3
  138. rasa/e2e_test/e2e_test_runner.py +5 -0
  139. rasa/e2e_test/e2e_test_schema.yml +3 -3
  140. rasa/model_manager/model_api.py +1 -1
  141. rasa/model_manager/socket_bridge.py +8 -2
  142. rasa/server.py +10 -0
  143. rasa/shared/agents/__init__.py +0 -0
  144. rasa/shared/agents/utils.py +35 -0
  145. rasa/shared/constants.py +5 -0
  146. rasa/shared/core/constants.py +12 -1
  147. rasa/shared/core/domain.py +5 -5
  148. rasa/shared/core/events.py +319 -0
  149. rasa/shared/core/flows/flows_list.py +2 -2
  150. rasa/shared/core/flows/flows_yaml_schema.json +101 -186
  151. rasa/shared/core/flows/steps/call.py +51 -5
  152. rasa/shared/core/flows/validation.py +45 -7
  153. rasa/shared/core/flows/yaml_flows_io.py +3 -3
  154. rasa/shared/providers/llm/_base_litellm_client.py +39 -7
  155. rasa/shared/providers/llm/litellm_router_llm_client.py +8 -4
  156. rasa/shared/providers/llm/llm_client.py +7 -3
  157. rasa/shared/providers/llm/llm_response.py +49 -0
  158. rasa/shared/providers/llm/self_hosted_llm_client.py +8 -4
  159. rasa/shared/utils/common.py +2 -1
  160. rasa/shared/utils/llm.py +28 -5
  161. rasa/shared/utils/mcp/__init__.py +0 -0
  162. rasa/shared/utils/mcp/server_connection.py +157 -0
  163. rasa/shared/utils/schemas/events.py +42 -0
  164. rasa/studio/upload.py +4 -7
  165. rasa/tracing/instrumentation/instrumentation.py +4 -2
  166. rasa/utils/common.py +53 -0
  167. rasa/utils/licensing.py +21 -10
  168. rasa/utils/plotting.py +1 -1
  169. rasa/version.py +1 -1
  170. {rasa_pro-3.13.7.dist-info → rasa_pro-3.14.0.dev1.dist-info}/METADATA +16 -15
  171. {rasa_pro-3.13.7.dist-info → rasa_pro-3.14.0.dev1.dist-info}/RECORD +174 -137
  172. rasa/core/channels/inspector/dist/assets/channel-51d02e9e.js +0 -1
  173. rasa/core/channels/inspector/dist/assets/clone-cc738fa6.js +0 -1
  174. rasa/core/channels/inspector/dist/assets/flowDiagram-v2-96b9c2cf-0c716443.js +0 -1
  175. rasa/core/channels/inspector/dist/assets/index-c804b295.js +0 -1335
  176. {rasa_pro-3.13.7.dist-info → rasa_pro-3.14.0.dev1.dist-info}/NOTICE +0 -0
  177. {rasa_pro-3.13.7.dist-info → rasa_pro-3.14.0.dev1.dist-info}/WHEEL +0 -0
  178. {rasa_pro-3.13.7.dist-info → rasa_pro-3.14.0.dev1.dist-info}/entry_points.txt +0 -0
@@ -40,6 +40,15 @@ export interface Stack {
40
40
  collect?: string
41
41
  utter?: string
42
42
  ended: boolean
43
+ agent_id?: string
44
+ state?: "waiting_for_input" | "interrupted"
45
+ }
46
+
47
+ export interface LatencyData {
48
+ rasa_processing_latency_ms?: number
49
+ asr_latency_ms?: number
50
+ tts_first_byte_latency_ms?: number
51
+ tts_complete_latency_ms?: number
43
52
  }
44
53
 
45
54
  export interface Tracker {
@@ -47,6 +56,7 @@ export interface Tracker {
47
56
  slots: { [key: string]: unknown }
48
57
  events: Event[]
49
58
  stack: Stack[]
59
+ latency?: LatencyData
50
60
  }
51
61
 
52
62
  export interface Flow {
@@ -87,4 +97,6 @@ interface Step {
87
97
  reset_after_flow_ends: boolean
88
98
  utter: string
89
99
  set_slots?: unknown
100
+ call?: string
101
+ noop?: boolean
90
102
  }
@@ -1,7 +1,10 @@
1
+ from __future__ import annotations
2
+
1
3
  import asyncio
2
4
  import audioop
3
5
  import base64
4
6
  import json
7
+ import time
5
8
  import uuid
6
9
  from functools import partial
7
10
  from typing import (
@@ -16,6 +19,7 @@ from typing import (
16
19
  Tuple,
17
20
  )
18
21
 
22
+ import orjson
19
23
  import structlog
20
24
 
21
25
  from rasa.core.channels import UserMessage
@@ -32,6 +36,7 @@ from rasa.core.channels.voice_stream.voice_channel import (
32
36
  VoiceInputChannel,
33
37
  VoiceOutputChannel,
34
38
  )
39
+ from rasa.core.exceptions import AgentNotReady
35
40
  from rasa.hooks import hookimpl
36
41
  from rasa.plugin import plugin_manager
37
42
  from rasa.shared.core.constants import ACTION_LISTEN_NAME
@@ -42,14 +47,16 @@ if TYPE_CHECKING:
42
47
  from sanic import Sanic, Websocket # type: ignore[attr-defined]
43
48
  from socketio import AsyncServer
44
49
 
45
- from rasa.core.channels.channel import InputChannel, UserMessage
50
+ from rasa.core.channels.channel import UserMessage
46
51
  from rasa.shared.core.trackers import DialogueStateTracker
47
52
 
48
53
 
49
54
  structlogger = structlog.get_logger()
50
55
 
51
56
 
52
- def tracker_as_dump(tracker: "DialogueStateTracker") -> str:
57
+ def tracker_as_dump(
58
+ tracker: "DialogueStateTracker", latency: Optional[float] = None
59
+ ) -> str:
53
60
  """Create a dump of the tracker state."""
54
61
  from rasa.shared.core.trackers import get_trackers_for_conversation_sessions
55
62
 
@@ -61,7 +68,10 @@ def tracker_as_dump(tracker: "DialogueStateTracker") -> str:
61
68
  last_tracker = multiple_tracker_sessions[-1]
62
69
 
63
70
  state = last_tracker.current_state(EventVerbosity.AFTER_RESTART)
64
- return json.dumps(state)
71
+
72
+ if latency is not None:
73
+ state["latency"] = {"rasa_processing_latency_ms": latency}
74
+ return orjson.dumps(state, option=orjson.OPT_SERIALIZE_NUMPY).decode("utf-8")
65
75
 
66
76
 
67
77
  def does_need_action_prediction(tracker: "DialogueStateTracker") -> bool:
@@ -175,11 +185,14 @@ class StudioChatInput(SocketIOInput, VoiceInputChannel):
175
185
  # `background_tasks` holds the asyncio tasks for voice streaming
176
186
  self.active_connections: Dict[str, SocketIOVoiceWebsocketAdapter] = {}
177
187
  self.background_tasks: Dict[str, asyncio.Task] = {}
188
+ self._turn_start_times: Dict[Text, float] = {}
178
189
 
179
190
  self._register_tracker_update_hook()
180
191
 
181
192
  @classmethod
182
- def from_credentials(cls, credentials: Optional[Dict[Text, Any]]) -> "InputChannel":
193
+ def from_credentials(
194
+ cls, credentials: Optional[Dict[Text, Any]]
195
+ ) -> "StudioChatInput":
183
196
  """Creates a StudioChatInput channel from credentials."""
184
197
  credentials = credentials or {}
185
198
 
@@ -199,19 +212,41 @@ class StudioChatInput(SocketIOInput, VoiceInputChannel):
199
212
  metadata_key=credentials.get("metadata_key", "metadata"),
200
213
  )
201
214
 
215
+ async def emit(self, event: str, data: str, room: str) -> None:
216
+ """Emits an event to the websocket."""
217
+ if not self.sio:
218
+ structlogger.error("studio_chat.emit.sio_not_initialized")
219
+ return
220
+ await self.sio.emit(event, data, room=room)
221
+
202
222
  def _register_tracker_update_hook(self) -> None:
203
223
  plugin_manager().register(StudioTrackerUpdatePlugin(self))
204
224
 
205
- async def on_tracker_updated(self, tracker: "DialogueStateTracker") -> None:
225
+ async def on_tracker_updated(
226
+ self, tracker: "DialogueStateTracker", latency: Optional[float] = None
227
+ ) -> None:
206
228
  """Triggers a tracker update notification after a change to the tracker."""
207
- await self.publish_tracker_update(tracker.sender_id, tracker_as_dump(tracker))
229
+ await self.publish_tracker_update(
230
+ tracker.sender_id, tracker_as_dump(tracker, latency)
231
+ )
208
232
 
209
- async def publish_tracker_update(self, sender_id: str, tracker_dump: Dict) -> None:
233
+ async def publish_tracker_update(self, sender_id: str, tracker_dump: str) -> None:
210
234
  """Publishes a tracker update notification to the websocket."""
211
- if not self.sio:
212
- structlogger.error("studio_chat.on_tracker_updated.sio_not_initialized")
213
- return
214
- await self.sio.emit("tracker", tracker_dump, room=sender_id)
235
+ await self.emit("tracker", tracker_dump, room=sender_id)
236
+
237
+ def _record_turn_start_time(self, sender_id: Text) -> None:
238
+ """Records the start time of a new turn."""
239
+ self._turn_start_times[sender_id] = time.time()
240
+
241
+ def _get_latency(self, sender_id: Text) -> Optional[float]:
242
+ """Returns the latency of the current turn in milliseconds."""
243
+ if sender_id not in self._turn_start_times:
244
+ return None
245
+
246
+ latency = (time.time() - self._turn_start_times[sender_id]) * 1000
247
+ # The turn is over, so we can remove the start time
248
+ del self._turn_start_times[sender_id]
249
+ return latency
215
250
 
216
251
  async def on_message_proxy(
217
252
  self,
@@ -222,10 +257,18 @@ class StudioChatInput(SocketIOInput, VoiceInputChannel):
222
257
 
223
258
  Triggers a tracker update notification after processing the message.
224
259
  """
260
+ self._record_turn_start_time(message.sender_id)
225
261
  await on_new_message(message)
226
262
 
227
- if not self.agent:
263
+ if not self.agent or not self.agent.is_ready():
228
264
  structlogger.error("studio_chat.on_message_proxy.agent_not_initialized")
265
+ await self.emit_error(
266
+ "The Rasa Pro model could not be loaded. "
267
+ "Please check the training and deployment logs "
268
+ "for more information.",
269
+ message.sender_id,
270
+ AgentNotReady("The Rasa Pro model could not be loaded."),
271
+ )
229
272
  return
230
273
 
231
274
  tracker = await self.agent.tracker_store.retrieve(message.sender_id)
@@ -233,7 +276,19 @@ class StudioChatInput(SocketIOInput, VoiceInputChannel):
233
276
  structlogger.error("studio_chat.on_message_proxy.tracker_not_found")
234
277
  return
235
278
 
236
- await self.on_tracker_updated(tracker)
279
+ latency = self._get_latency(message.sender_id)
280
+ await self.on_tracker_updated(tracker, latency)
281
+
282
+ async def emit_error(self, message: str, room: str, e: Exception) -> None:
283
+ await self.emit(
284
+ "error",
285
+ {
286
+ "message": message,
287
+ "error": str(e),
288
+ "exception": str(type(e).__name__),
289
+ },
290
+ room=room,
291
+ )
237
292
 
238
293
  async def handle_tracker_update(self, sid: str, data: Dict) -> None:
239
294
  from rasa.shared.core.trackers import DialogueStateTracker
@@ -251,21 +306,41 @@ class StudioChatInput(SocketIOInput, VoiceInputChannel):
251
306
  structlogger.error("studio_chat.sio.domain_not_initialized")
252
307
  return None
253
308
 
254
- async with self.agent.lock_store.lock(data["sender_id"]):
255
- tracker = DialogueStateTracker.from_dict(
256
- data["sender_id"], data["events"], domain.slots
257
- )
258
-
259
- # will override an existing tracker with the same id!
260
- await self.agent.tracker_store.save(tracker)
309
+ tracker: Optional[DialogueStateTracker] = None
261
310
 
262
- processor = self.agent.processor
263
- if processor and does_need_action_prediction(tracker):
264
- output_channel = self.get_output_channel()
311
+ async with self.agent.lock_store.lock(data["sender_id"]):
312
+ try:
313
+ tracker = DialogueStateTracker.from_dict(
314
+ data["sender_id"], data["events"], domain.slots
315
+ )
265
316
 
266
- await processor._run_prediction_loop(output_channel, tracker)
317
+ # will override an existing tracker with the same id!
267
318
  await self.agent.tracker_store.save(tracker)
268
319
 
320
+ processor = self.agent.processor
321
+ if processor and does_need_action_prediction(tracker):
322
+ output_channel = self.get_output_channel()
323
+
324
+ await processor._run_prediction_loop(output_channel, tracker)
325
+ await self.agent.tracker_store.save(tracker)
326
+ except Exception as e:
327
+ structlogger.error(
328
+ "studio_chat.sio.handle_tracker_update.error",
329
+ error=e,
330
+ sender_id=data["sender_id"],
331
+ )
332
+ await self.emit_error(
333
+ "An error occurred while updating the conversation.",
334
+ data["sender_id"],
335
+ e,
336
+ )
337
+
338
+ if not tracker:
339
+ # in case the tracker couldn't be updated, we retrieve the prior
340
+ # version and use that to populate the update
341
+ tracker = await self.agent.tracker_store.get_or_create_tracker(
342
+ data["sender_id"]
343
+ )
269
344
  await self.on_tracker_updated(tracker)
270
345
 
271
346
  def channel_bytes_to_rasa_audio_bytes(self, input_bytes: bytes) -> RasaAudioBytes:
@@ -275,7 +350,7 @@ class StudioChatInput(SocketIOInput, VoiceInputChannel):
275
350
  async def collect_call_parameters(
276
351
  self, channel_websocket: "Websocket"
277
352
  ) -> Optional[CallParameters]:
278
- """Voice method to collect call parameters"""
353
+ """Voice method to collect call parameters."""
279
354
  session_id = channel_websocket.session_id
280
355
  return CallParameters(session_id, "local", "local", stream_id=session_id)
281
356
 
@@ -292,20 +367,20 @@ class StudioChatInput(SocketIOInput, VoiceInputChannel):
292
367
  elif "marker" in message:
293
368
  if message["marker"] == call_state.latest_bot_audio_id:
294
369
  # Just finished streaming last audio bytes
295
- call_state.is_bot_speaking = False # type: ignore[attr-defined]
370
+ call_state.is_bot_speaking = False
296
371
  if call_state.should_hangup:
297
372
  structlogger.debug(
298
373
  "studio_chat.hangup", marker=call_state.latest_bot_audio_id
299
374
  )
300
375
  return EndConversationAction()
301
376
  else:
302
- call_state.is_bot_speaking = True # type: ignore[attr-defined]
377
+ call_state.is_bot_speaking = True
303
378
  return ContinueConversationAction()
304
379
 
305
380
  def create_output_channel(
306
381
  self, voice_websocket: "Websocket", tts_engine: TTSEngine
307
382
  ) -> VoiceOutputChannel:
308
- """Create a voice output channel"""
383
+ """Create a voice output channel."""
309
384
  return StudioVoiceOutputChannel(
310
385
  voice_websocket,
311
386
  tts_engine,
@@ -382,9 +457,8 @@ class StudioChatInput(SocketIOInput, VoiceInputChannel):
382
457
  def blueprint(
383
458
  self, on_new_message: Callable[["UserMessage"], Awaitable[Any]]
384
459
  ) -> SocketBlueprint:
385
- socket_blueprint = super().blueprint(
386
- partial(self.on_message_proxy, on_new_message)
387
- )
460
+ proxied_on_message = partial(self.on_message_proxy, on_new_message)
461
+ socket_blueprint = super().blueprint(proxied_on_message)
388
462
 
389
463
  if not self.sio:
390
464
  structlogger.error("studio_chat.blueprint.sio_not_initialized")
@@ -419,7 +493,7 @@ class StudioChatInput(SocketIOInput, VoiceInputChannel):
419
493
 
420
494
  # start a voice session if requested
421
495
  if data and data.get("is_voice", False):
422
- self._start_voice_session(data["session_id"], sid, on_new_message)
496
+ self._start_voice_session(data["session_id"], sid, proxied_on_message)
423
497
 
424
498
  @self.sio.on(self.user_message_evt, namespace=self.namespace)
425
499
  async def handle_message(sid: Text, data: Dict) -> None:
@@ -433,7 +507,7 @@ class StudioChatInput(SocketIOInput, VoiceInputChannel):
433
507
  return
434
508
 
435
509
  # Handle text messages
436
- await self.handle_user_message(sid, data, on_new_message)
510
+ await self.handle_user_message(sid, data, proxied_on_message)
437
511
 
438
512
  @self.sio.on("update_tracker", namespace=self.namespace)
439
513
  async def on_update_tracker(sid: Text, data: Dict) -> None:
@@ -457,7 +531,24 @@ class StudioVoiceOutputChannel(VoiceOutputChannel):
457
531
 
458
532
  def create_marker_message(self, recipient_id: str) -> Tuple[str, str]:
459
533
  message_id = uuid.uuid4().hex
460
- return json.dumps({"marker": message_id}), message_id
534
+ marker_data = {"marker": message_id}
535
+
536
+ # Include comprehensive latency information if available
537
+ latency_data = {
538
+ "asr_latency_ms": call_state.asr_latency_ms,
539
+ "rasa_processing_latency_ms": call_state.rasa_processing_latency_ms,
540
+ "tts_first_byte_latency_ms": call_state.tts_first_byte_latency_ms,
541
+ "tts_complete_latency_ms": call_state.tts_complete_latency_ms,
542
+ }
543
+
544
+ # Filter out None values from latency data
545
+ latency_data = {k: v for k, v in latency_data.items() if v is not None}
546
+
547
+ # Add latency data to marker if any metrics are available
548
+ if latency_data:
549
+ marker_data["latency"] = latency_data # type: ignore[assignment]
550
+
551
+ return json.dumps(marker_data), message_id
461
552
 
462
553
 
463
554
  class SocketIOVoiceWebsocketAdapter:
@@ -30,7 +30,7 @@ TWILIO_VOICE_PATH = "webhooks/twilio_voice/webhook"
30
30
 
31
31
 
32
32
  def map_call_params(form: RequestParameters) -> CallParameters:
33
- """Map the Audiocodes parameters to the CallParameters dataclass."""
33
+ """Map the Twilio Voice parameters to the CallParameters dataclass."""
34
34
  return CallParameters(
35
35
  call_id=form.get("CallSid"),
36
36
  user_phone=form.get("Caller"),
@@ -1,3 +1,5 @@
1
+ from __future__ import annotations
2
+
1
3
  import asyncio
2
4
  import base64
3
5
  import hmac
@@ -21,6 +23,7 @@ from rasa.core.channels.voice_stream.call_state import (
21
23
  call_state,
22
24
  )
23
25
  from rasa.core.channels.voice_stream.tts.tts_engine import TTSEngine
26
+ from rasa.core.channels.voice_stream.util import repack_voice_credentials
24
27
  from rasa.core.channels.voice_stream.voice_channel import (
25
28
  ContinueConversationAction,
26
29
  EndConversationAction,
@@ -85,7 +88,7 @@ class AudiocodesVoiceOutputChannel(VoiceOutputChannel):
85
88
  # however, Audiocodes does not have an event to indicate that.
86
89
  # This is an approximation, as the bot will be sent the audio chunks next
87
90
  # which are played to the user immediately.
88
- call_state.is_bot_speaking = True # type: ignore[attr-defined]
91
+ call_state.is_bot_speaking = True
89
92
 
90
93
  async def send_intermediate_marker(self, recipient_id: str) -> None:
91
94
  """Audiocodes doesn't need intermediate markers, so do nothing."""
@@ -127,10 +130,10 @@ class AudiocodesVoiceInputChannel(VoiceInputChannel):
127
130
  def from_credentials(
128
131
  cls,
129
132
  credentials: Optional[Dict[str, Any]],
130
- ) -> "AudiocodesVoiceInputChannel":
131
- channel = super().from_credentials(credentials)
132
- channel.token = credentials.get("token") # type: ignore[attr-defined, union-attr]
133
- return channel # type: ignore[return-value]
133
+ ) -> AudiocodesVoiceInputChannel:
134
+ cls.validate_basic_credentials(credentials)
135
+ new_creds = repack_voice_credentials(credentials)
136
+ return cls(**new_creds)
134
137
 
135
138
  def channel_bytes_to_rasa_audio_bytes(self, input_bytes: bytes) -> RasaAudioBytes:
136
139
  return RasaAudioBytes(base64.b64decode(input_bytes))
@@ -184,7 +187,7 @@ class AudiocodesVoiceInputChannel(VoiceInputChannel):
184
187
  pass
185
188
  elif activity["name"] == "playFinished":
186
189
  logger.debug("audiocodes_stream.playFinished", data=activity)
187
- call_state.is_bot_speaking = False # type: ignore[attr-defined]
190
+ call_state.is_bot_speaking = False
188
191
  if call_state.should_hangup:
189
192
  logger.info("audiocodes_stream.hangup")
190
193
  self._send_hangup(ws, data)
@@ -1,8 +1,10 @@
1
+ from __future__ import annotations
2
+
1
3
  import audioop
2
4
  import base64
3
5
  import json
4
6
  import uuid
5
- from typing import Any, Awaitable, Callable, Optional, Tuple
7
+ from typing import Any, Awaitable, Callable, Dict, Optional, Tuple
6
8
 
7
9
  import structlog
8
10
  from sanic import ( # type: ignore[attr-defined]
@@ -18,6 +20,7 @@ from rasa.core.channels.voice_ready.utils import CallParameters
18
20
  from rasa.core.channels.voice_stream.audio_bytes import RasaAudioBytes
19
21
  from rasa.core.channels.voice_stream.call_state import call_state
20
22
  from rasa.core.channels.voice_stream.tts.tts_engine import TTSEngine
23
+ from rasa.core.channels.voice_stream.util import repack_voice_credentials
21
24
  from rasa.core.channels.voice_stream.voice_channel import (
22
25
  ContinueConversationAction,
23
26
  EndConversationAction,
@@ -45,10 +48,33 @@ class BrowserAudioOutputChannel(VoiceOutputChannel):
45
48
 
46
49
  def create_marker_message(self, recipient_id: str) -> Tuple[str, str]:
47
50
  message_id = uuid.uuid4().hex
48
- return json.dumps({"marker": message_id}), message_id
51
+ marker_data = {"marker": message_id}
52
+
53
+ # Include comprehensive latency information if available
54
+ latency_data = {
55
+ "asr_latency_ms": call_state.asr_latency_ms,
56
+ "rasa_processing_latency_ms": call_state.rasa_processing_latency_ms,
57
+ "tts_first_byte_latency_ms": call_state.tts_first_byte_latency_ms,
58
+ "tts_complete_latency_ms": call_state.tts_complete_latency_ms,
59
+ }
60
+
61
+ # Filter out None values from latency data
62
+ latency_data = {k: v for k, v in latency_data.items() if v is not None}
63
+
64
+ # Add latency data to marker if any metrics are available
65
+ if latency_data:
66
+ marker_data["latency"] = latency_data # type: ignore[assignment]
67
+
68
+ return json.dumps(marker_data), message_id
49
69
 
50
70
 
51
71
  class BrowserAudioInputChannel(VoiceInputChannel):
72
+ def __init__(
73
+ self, server_url: str, asr_config: Dict[str, Any], tts_config: Dict[str, Any]
74
+ ) -> None:
75
+ """Initializes the browser audio input channel."""
76
+ super().__init__(server_url, asr_config, tts_config)
77
+
52
78
  @classmethod
53
79
  def name(cls) -> str:
54
80
  return "browser_audio"
@@ -62,6 +88,15 @@ class BrowserAudioInputChannel(VoiceInputChannel):
62
88
  call_id = f"inspect-{uuid.uuid4()}"
63
89
  return CallParameters(call_id, "local", "local", stream_id=call_id)
64
90
 
91
+ @classmethod
92
+ def from_credentials(
93
+ cls,
94
+ credentials: Optional[Dict[str, Any]],
95
+ ) -> BrowserAudioInputChannel:
96
+ cls.validate_basic_credentials(credentials)
97
+ new_creds = repack_voice_credentials(credentials)
98
+ return cls(**new_creds)
99
+
65
100
  def map_input_message(
66
101
  self,
67
102
  message: Any,
@@ -75,14 +110,14 @@ class BrowserAudioInputChannel(VoiceInputChannel):
75
110
  elif "marker" in data:
76
111
  if data["marker"] == call_state.latest_bot_audio_id:
77
112
  # Just finished streaming last audio bytes
78
- call_state.is_bot_speaking = False # type: ignore[attr-defined]
113
+ call_state.is_bot_speaking = False
79
114
  if call_state.should_hangup:
80
115
  logger.debug(
81
116
  "browser_audio.hangup", marker=call_state.latest_bot_audio_id
82
117
  )
83
118
  return EndConversationAction()
84
119
  else:
85
- call_state.is_bot_speaking = True # type: ignore[attr-defined]
120
+ call_state.is_bot_speaking = True
86
121
  return ContinueConversationAction()
87
122
 
88
123
  def create_output_channel(
@@ -1,7 +1,7 @@
1
1
  import asyncio
2
2
  from contextvars import ContextVar
3
3
  from dataclasses import dataclass, field
4
- from typing import Any, Dict, Optional
4
+ from typing import Any, Dict, Optional, cast
5
5
 
6
6
  from werkzeug.local import LocalProxy
7
7
 
@@ -19,9 +19,20 @@ class CallState:
19
19
  should_hangup: bool = False
20
20
  connection_failed: bool = False
21
21
 
22
+ # Latency tracking - start times only
23
+ user_speech_start_time: Optional[float] = None
24
+ rasa_processing_start_time: Optional[float] = None
25
+ tts_start_time: Optional[float] = None
26
+
27
+ # Calculated latencies (used by channels like browser_audio)
28
+ asr_latency_ms: Optional[float] = None
29
+ rasa_processing_latency_ms: Optional[float] = None
30
+ tts_first_byte_latency_ms: Optional[float] = None
31
+ tts_complete_latency_ms: Optional[float] = None
32
+
22
33
  # Generic field for channel-specific state data
23
34
  channel_data: Dict[str, Any] = field(default_factory=dict)
24
35
 
25
36
 
26
37
  _call_state: ContextVar[CallState] = ContextVar("call_state")
27
- call_state = LocalProxy(_call_state)
38
+ call_state: CallState = cast(CallState, LocalProxy(_call_state))
@@ -1,3 +1,5 @@
1
+ from __future__ import annotations
2
+
1
3
  import asyncio
2
4
  import base64
3
5
  import hashlib
@@ -21,6 +23,7 @@ from rasa.core.channels.voice_stream.call_state import (
21
23
  call_state,
22
24
  )
23
25
  from rasa.core.channels.voice_stream.tts.tts_engine import TTSEngine
26
+ from rasa.core.channels.voice_stream.util import repack_voice_credentials
24
27
  from rasa.core.channels.voice_stream.voice_channel import (
25
28
  ContinueConversationAction,
26
29
  EndConversationAction,
@@ -54,7 +57,7 @@ logger = structlog.get_logger(__name__)
54
57
 
55
58
 
56
59
  def map_call_params(data: Dict[Text, Any]) -> CallParameters:
57
- """Map the twilio stream parameters to the CallParameters dataclass."""
60
+ """Map the Genesys parameters to the CallParameters dataclass."""
58
61
  parameters = data["parameters"]
59
62
  participant = parameters["participant"]
60
63
  # sent as {"ani": "tel:+491604697810"}
@@ -107,7 +110,7 @@ class GenesysInputChannel(VoiceInputChannel):
107
110
  def from_credentials(
108
111
  cls,
109
112
  credentials: Optional[Dict[str, Any]],
110
- ) -> "GenesysInputChannel":
113
+ ) -> GenesysInputChannel:
111
114
  """Create a channel from credentials dictionary.
112
115
 
113
116
  Args:
@@ -121,21 +124,21 @@ class GenesysInputChannel(VoiceInputChannel):
121
124
  Returns:
122
125
  GenesysInputChannel instance
123
126
  """
124
- channel = super().from_credentials(credentials)
127
+ cls.validate_credentials(credentials)
128
+ new_creds = repack_voice_credentials(credentials)
129
+ return cls(**new_creds)
125
130
 
126
- # Check required Genesys-specific credentials
131
+ @classmethod
132
+ def validate_credentials(cls, credentials: Optional[Dict[str, Any]]) -> None:
133
+ """Validate the credentials for the Genesys voice channel."""
134
+ cls.validate_basic_credentials(credentials)
127
135
  if not credentials.get("api_key"): # type: ignore[union-attr]
128
136
  raise InvalidConfigException(
129
137
  "No API key given for Genesys voice channel (api_key)."
130
138
  )
131
139
 
132
- # Update channel with Genesys-specific credentials
133
- channel.api_key = credentials["api_key"] # type: ignore[index,attr-defined]
134
- channel.client_secret = credentials.get("client_secret") # type: ignore[union-attr,attr-defined]
135
-
136
- return channel # type: ignore[return-value]
137
-
138
- def _ensure_channel_data_initialized(self) -> None:
140
+ @staticmethod
141
+ def _ensure_channel_data_initialized() -> None:
139
142
  """Initialize Genesys-specific channel data if not already present.
140
143
 
141
144
  Genesys requires the server and client each maintain a
@@ -216,10 +219,10 @@ class GenesysInputChannel(VoiceInputChannel):
216
219
  self.handle_ping(ws, data)
217
220
  elif msg_type == "playback_started":
218
221
  logger.debug("genesys.handle_playback_started", message=data)
219
- call_state.is_bot_speaking = True # type: ignore[attr-defined]
222
+ call_state.is_bot_speaking = True
220
223
  elif msg_type == "playback_completed":
221
224
  logger.debug("genesys.handle_playback_completed", message=data)
222
- call_state.is_bot_speaking = False # type: ignore[attr-defined]
225
+ call_state.is_bot_speaking = False
223
226
  if call_state.should_hangup:
224
227
  logger.info("genesys.hangup")
225
228
  self.disconnect(ws, data)
@@ -1,3 +1,5 @@
1
+ from __future__ import annotations
2
+
1
3
  import audioop
2
4
  import json
3
5
  import uuid
@@ -20,6 +22,7 @@ from rasa.core.channels.voice_ready.utils import (
20
22
  from rasa.core.channels.voice_stream.audio_bytes import RasaAudioBytes
21
23
  from rasa.core.channels.voice_stream.call_state import call_state
22
24
  from rasa.core.channels.voice_stream.tts.tts_engine import TTSEngine
25
+ from rasa.core.channels.voice_stream.util import repack_voice_credentials
23
26
  from rasa.core.channels.voice_stream.voice_channel import (
24
27
  ContinueConversationAction,
25
28
  EndConversationAction,
@@ -35,7 +38,7 @@ JAMBONZ_STREAMS_WEBSOCKET_PATH = "webhooks/jambonz_stream/websocket"
35
38
 
36
39
 
37
40
  def map_call_params(data: Dict[Text, str]) -> CallParameters:
38
- """Map the twilio stream parameters to the CallParameters dataclass."""
41
+ """Map the Jambonz stream parameters to the CallParameters dataclass."""
39
42
  call_sid = data.get("callSid", "None")
40
43
  from_number = data.get("from", "Unknown")
41
44
  to_number = data.get("to")
@@ -94,7 +97,7 @@ class JambonzStreamInputChannel(VoiceInputChannel):
94
97
  @classmethod
95
98
  def from_credentials(
96
99
  cls, credentials: Optional[Dict[Text, Any]]
97
- ) -> "JambonzStreamInputChannel":
100
+ ) -> JambonzStreamInputChannel:
98
101
  """Create a channel from credentials dictionary.
99
102
 
100
103
  Args:
@@ -109,19 +112,18 @@ class JambonzStreamInputChannel(VoiceInputChannel):
109
112
  JambonzStreamInputChannel instance
110
113
  """
111
114
  # Get common credentials from parent
112
- channel = super().from_credentials(credentials)
115
+ cls.validate_credentials(credentials)
116
+ new_creds = repack_voice_credentials(credentials)
117
+ return cls(**new_creds)
113
118
 
119
+ @classmethod
120
+ def validate_credentials(cls, credentials: Optional[Dict[Text, Any]]) -> None:
121
+ cls.validate_basic_credentials(credentials)
114
122
  # Check optional basic auth credentials
115
123
  username = credentials.get("username") # type: ignore[union-attr]
116
124
  password = credentials.get("password") # type: ignore[union-attr]
117
125
  validate_username_password_credentials(username, password, "Jambonz Stream")
118
126
 
119
- # Update channel with auth credentials
120
- channel.username = username # type: ignore[attr-defined]
121
- channel.password = password # type: ignore[attr-defined]
122
-
123
- return channel # type: ignore[return-value]
124
-
125
127
  def _websocket_stream_url(self) -> str:
126
128
  """Returns the websocket stream URL."""
127
129
  # depending on the config value, the url might contain http as a
@@ -158,14 +160,14 @@ class JambonzStreamInputChannel(VoiceInputChannel):
158
160
  if data["type"] == "mark":
159
161
  if data["data"]["name"] == call_state.latest_bot_audio_id:
160
162
  # Just finished streaming last audio bytes
161
- call_state.is_bot_speaking = False # type: ignore[attr-defined]
163
+ call_state.is_bot_speaking = False
162
164
  if call_state.should_hangup:
163
165
  logger.debug(
164
166
  "jambonz.hangup", marker=call_state.latest_bot_audio_id
165
167
  )
166
168
  return EndConversationAction()
167
169
  else:
168
- call_state.is_bot_speaking = True # type: ignore[attr-defined]
170
+ call_state.is_bot_speaking = True
169
171
  elif data["event"] == "dtmf":
170
172
  # TODO: handle DTMF input
171
173
  logger.debug("jambonz.dtmf.received", dtmf=data["dtmf"])