rasa-pro 3.11.0__py3-none-any.whl → 3.11.0a1__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.

Potentially problematic release.


This version of rasa-pro might be problematic. Click here for more details.

Files changed (220) hide show
  1. README.md +396 -17
  2. rasa/__main__.py +15 -31
  3. rasa/api.py +1 -5
  4. rasa/cli/arguments/default_arguments.py +2 -1
  5. rasa/cli/arguments/shell.py +1 -5
  6. rasa/cli/arguments/train.py +0 -14
  7. rasa/cli/e2e_test.py +1 -1
  8. rasa/cli/evaluate.py +8 -8
  9. rasa/cli/inspect.py +7 -15
  10. rasa/cli/interactive.py +0 -1
  11. rasa/cli/llm_fine_tuning.py +1 -1
  12. rasa/cli/project_templates/calm/config.yml +7 -5
  13. rasa/cli/project_templates/calm/endpoints.yml +2 -15
  14. rasa/cli/project_templates/tutorial/config.yml +5 -8
  15. rasa/cli/project_templates/tutorial/data/flows.yml +1 -1
  16. rasa/cli/project_templates/tutorial/data/patterns.yml +0 -5
  17. rasa/cli/project_templates/tutorial/domain.yml +0 -14
  18. rasa/cli/project_templates/tutorial/endpoints.yml +0 -5
  19. rasa/cli/run.py +1 -1
  20. rasa/cli/scaffold.py +2 -4
  21. rasa/cli/studio/studio.py +8 -18
  22. rasa/cli/studio/upload.py +15 -0
  23. rasa/cli/train.py +0 -3
  24. rasa/cli/utils.py +1 -6
  25. rasa/cli/x.py +8 -8
  26. rasa/constants.py +1 -3
  27. rasa/core/actions/action.py +33 -75
  28. rasa/core/actions/e2e_stub_custom_action_executor.py +1 -5
  29. rasa/core/actions/http_custom_action_executor.py +0 -4
  30. rasa/core/channels/__init__.py +0 -2
  31. rasa/core/channels/channel.py +0 -20
  32. rasa/core/channels/development_inspector.py +3 -10
  33. rasa/core/channels/inspector/dist/assets/{arc-bc141fb2.js → arc-86942a71.js} +1 -1
  34. rasa/core/channels/inspector/dist/assets/{c4Diagram-d0fbc5ce-be2db283.js → c4Diagram-d0fbc5ce-b0290676.js} +1 -1
  35. rasa/core/channels/inspector/dist/assets/{classDiagram-936ed81e-55366915.js → classDiagram-936ed81e-f6405f6e.js} +1 -1
  36. rasa/core/channels/inspector/dist/assets/{classDiagram-v2-c3cb15f1-bb529518.js → classDiagram-v2-c3cb15f1-ef61ac77.js} +1 -1
  37. rasa/core/channels/inspector/dist/assets/{createText-62fc7601-b0ec81d6.js → createText-62fc7601-f0411e58.js} +1 -1
  38. rasa/core/channels/inspector/dist/assets/{edges-f2ad444c-6166330c.js → edges-f2ad444c-7dcc4f3b.js} +1 -1
  39. rasa/core/channels/inspector/dist/assets/{erDiagram-9d236eb7-5ccc6a8e.js → erDiagram-9d236eb7-e0c092d7.js} +1 -1
  40. rasa/core/channels/inspector/dist/assets/{flowDb-1972c806-fca3bfe4.js → flowDb-1972c806-fba2e3ce.js} +1 -1
  41. rasa/core/channels/inspector/dist/assets/{flowDiagram-7ea5b25a-4739080f.js → flowDiagram-7ea5b25a-7a70b71a.js} +1 -1
  42. rasa/core/channels/inspector/dist/assets/flowDiagram-v2-855bc5b3-24a5f41a.js +1 -0
  43. rasa/core/channels/inspector/dist/assets/{flowchart-elk-definition-abe16c3d-7c1b0e0f.js → flowchart-elk-definition-abe16c3d-00a59b68.js} +1 -1
  44. rasa/core/channels/inspector/dist/assets/{ganttDiagram-9b5ea136-772fd050.js → ganttDiagram-9b5ea136-293c91fa.js} +1 -1
  45. rasa/core/channels/inspector/dist/assets/{gitGraphDiagram-99d0ae7c-8eae1dc9.js → gitGraphDiagram-99d0ae7c-07b2d68c.js} +1 -1
  46. rasa/core/channels/inspector/dist/assets/{index-2c4b9a3b-f55afcdf.js → index-2c4b9a3b-bc959fbd.js} +1 -1
  47. rasa/core/channels/inspector/dist/assets/{index-e7cef9de.js → index-3a8a5a28.js} +143 -143
  48. rasa/core/channels/inspector/dist/assets/{infoDiagram-736b4530-124d4a14.js → infoDiagram-736b4530-4a350f72.js} +1 -1
  49. rasa/core/channels/inspector/dist/assets/{journeyDiagram-df861f2b-7c4fae44.js → journeyDiagram-df861f2b-af464fb7.js} +1 -1
  50. rasa/core/channels/inspector/dist/assets/{layout-b9885fb6.js → layout-0071f036.js} +1 -1
  51. rasa/core/channels/inspector/dist/assets/{line-7c59abb6.js → line-2f73cc83.js} +1 -1
  52. rasa/core/channels/inspector/dist/assets/{linear-4776f780.js → linear-f014b4cc.js} +1 -1
  53. rasa/core/channels/inspector/dist/assets/{mindmap-definition-beec6740-2332c46c.js → mindmap-definition-beec6740-d2426fb6.js} +1 -1
  54. rasa/core/channels/inspector/dist/assets/{pieDiagram-dbbf0591-8fb39303.js → pieDiagram-dbbf0591-776f01a2.js} +1 -1
  55. rasa/core/channels/inspector/dist/assets/{quadrantDiagram-4d7f4fd6-3c7180a2.js → quadrantDiagram-4d7f4fd6-82e00b57.js} +1 -1
  56. rasa/core/channels/inspector/dist/assets/{requirementDiagram-6fc4c22a-e910bcb8.js → requirementDiagram-6fc4c22a-ea13c6bb.js} +1 -1
  57. rasa/core/channels/inspector/dist/assets/{sankeyDiagram-8f13d901-ead16c89.js → sankeyDiagram-8f13d901-1feca7e9.js} +1 -1
  58. rasa/core/channels/inspector/dist/assets/{sequenceDiagram-b655622a-29a02a19.js → sequenceDiagram-b655622a-070c61d2.js} +1 -1
  59. rasa/core/channels/inspector/dist/assets/{stateDiagram-59f0c015-042b3137.js → stateDiagram-59f0c015-24f46263.js} +1 -1
  60. rasa/core/channels/inspector/dist/assets/{stateDiagram-v2-2b26beab-2178c0f3.js → stateDiagram-v2-2b26beab-c9056051.js} +1 -1
  61. rasa/core/channels/inspector/dist/assets/{styles-080da4f6-23ffa4fc.js → styles-080da4f6-08abc34a.js} +1 -1
  62. rasa/core/channels/inspector/dist/assets/{styles-3dcbcfbf-94f59763.js → styles-3dcbcfbf-bc74c25a.js} +1 -1
  63. rasa/core/channels/inspector/dist/assets/{styles-9c745c82-78a6bebc.js → styles-9c745c82-4e5d66de.js} +1 -1
  64. rasa/core/channels/inspector/dist/assets/{svgDrawCommon-4835440b-eae2a6f6.js → svgDrawCommon-4835440b-849c4517.js} +1 -1
  65. rasa/core/channels/inspector/dist/assets/{timeline-definition-5b62e21b-5c968d92.js → timeline-definition-5b62e21b-d0fb1598.js} +1 -1
  66. rasa/core/channels/inspector/dist/assets/{xychartDiagram-2b33534f-fd3db0d5.js → xychartDiagram-2b33534f-04d115e2.js} +1 -1
  67. rasa/core/channels/inspector/dist/index.html +1 -1
  68. rasa/core/channels/inspector/src/App.tsx +1 -1
  69. rasa/core/channels/inspector/src/components/LoadingSpinner.tsx +3 -6
  70. rasa/core/channels/socketio.py +2 -7
  71. rasa/core/channels/telegram.py +1 -1
  72. rasa/core/channels/twilio.py +1 -1
  73. rasa/core/channels/voice_ready/audiocodes.py +4 -15
  74. rasa/core/channels/voice_ready/jambonz.py +4 -15
  75. rasa/core/channels/voice_ready/twilio_voice.py +21 -6
  76. rasa/core/channels/voice_ready/utils.py +5 -6
  77. rasa/core/channels/voice_stream/asr/asr_engine.py +1 -19
  78. rasa/core/channels/voice_stream/asr/asr_event.py +0 -5
  79. rasa/core/channels/voice_stream/asr/deepgram.py +15 -28
  80. rasa/core/channels/voice_stream/audio_bytes.py +0 -1
  81. rasa/core/channels/voice_stream/tts/azure.py +3 -9
  82. rasa/core/channels/voice_stream/tts/cartesia.py +8 -12
  83. rasa/core/channels/voice_stream/tts/tts_engine.py +1 -11
  84. rasa/core/channels/voice_stream/twilio_media_streams.py +19 -28
  85. rasa/core/channels/voice_stream/util.py +4 -4
  86. rasa/core/channels/voice_stream/voice_channel.py +42 -222
  87. rasa/core/featurizers/single_state_featurizer.py +1 -22
  88. rasa/core/featurizers/tracker_featurizers.py +18 -115
  89. rasa/core/information_retrieval/qdrant.py +0 -1
  90. rasa/core/nlg/contextual_response_rephraser.py +25 -44
  91. rasa/core/persistor.py +34 -191
  92. rasa/core/policies/enterprise_search_policy.py +60 -119
  93. rasa/core/policies/flows/flow_executor.py +4 -7
  94. rasa/core/policies/intentless_policy.py +22 -82
  95. rasa/core/policies/ted_policy.py +33 -58
  96. rasa/core/policies/unexpected_intent_policy.py +7 -15
  97. rasa/core/processor.py +13 -89
  98. rasa/core/run.py +2 -2
  99. rasa/core/training/interactive.py +35 -34
  100. rasa/core/utils.py +22 -58
  101. rasa/dialogue_understanding/coexistence/llm_based_router.py +12 -39
  102. rasa/dialogue_understanding/commands/__init__.py +0 -4
  103. rasa/dialogue_understanding/commands/change_flow_command.py +0 -6
  104. rasa/dialogue_understanding/commands/utils.py +0 -5
  105. rasa/dialogue_understanding/generator/constants.py +0 -2
  106. rasa/dialogue_understanding/generator/flow_retrieval.py +4 -49
  107. rasa/dialogue_understanding/generator/llm_based_command_generator.py +23 -37
  108. rasa/dialogue_understanding/generator/multi_step/multi_step_llm_command_generator.py +10 -57
  109. rasa/dialogue_understanding/generator/nlu_command_adapter.py +1 -19
  110. rasa/dialogue_understanding/generator/single_step/command_prompt_template.jinja2 +0 -3
  111. rasa/dialogue_understanding/generator/single_step/single_step_llm_command_generator.py +10 -90
  112. rasa/dialogue_understanding/patterns/default_flows_for_patterns.yml +0 -53
  113. rasa/dialogue_understanding/processor/command_processor.py +1 -21
  114. rasa/e2e_test/assertions.py +16 -133
  115. rasa/e2e_test/assertions_schema.yml +0 -23
  116. rasa/e2e_test/e2e_test_case.py +6 -85
  117. rasa/e2e_test/e2e_test_runner.py +4 -6
  118. rasa/e2e_test/utils/io.py +1 -3
  119. rasa/engine/loader.py +0 -12
  120. rasa/engine/validation.py +11 -541
  121. rasa/keys +1 -0
  122. rasa/llm_fine_tuning/notebooks/unsloth_finetuning.ipynb +407 -0
  123. rasa/model_training.py +7 -29
  124. rasa/nlu/classifiers/diet_classifier.py +25 -38
  125. rasa/nlu/classifiers/logistic_regression_classifier.py +9 -22
  126. rasa/nlu/classifiers/sklearn_intent_classifier.py +16 -37
  127. rasa/nlu/extractors/crf_entity_extractor.py +50 -93
  128. rasa/nlu/featurizers/sparse_featurizer/count_vectors_featurizer.py +16 -45
  129. rasa/nlu/featurizers/sparse_featurizer/lexical_syntactic_featurizer.py +17 -52
  130. rasa/nlu/featurizers/sparse_featurizer/regex_featurizer.py +3 -5
  131. rasa/nlu/tokenizers/whitespace_tokenizer.py +14 -3
  132. rasa/server.py +1 -3
  133. rasa/shared/constants.py +0 -61
  134. rasa/shared/core/constants.py +0 -9
  135. rasa/shared/core/domain.py +5 -8
  136. rasa/shared/core/flows/flow.py +0 -5
  137. rasa/shared/core/flows/flows_list.py +1 -5
  138. rasa/shared/core/flows/flows_yaml_schema.json +0 -10
  139. rasa/shared/core/flows/validation.py +0 -96
  140. rasa/shared/core/flows/yaml_flows_io.py +4 -13
  141. rasa/shared/core/slots.py +0 -5
  142. rasa/shared/importers/importer.py +2 -19
  143. rasa/shared/importers/rasa.py +1 -5
  144. rasa/shared/nlu/training_data/features.py +2 -120
  145. rasa/shared/nlu/training_data/formats/rasa_yaml.py +3 -18
  146. rasa/shared/providers/_configs/azure_openai_client_config.py +3 -5
  147. rasa/shared/providers/_configs/openai_client_config.py +1 -1
  148. rasa/shared/providers/_configs/self_hosted_llm_client_config.py +0 -1
  149. rasa/shared/providers/_configs/utils.py +0 -16
  150. rasa/shared/providers/embedding/_base_litellm_embedding_client.py +29 -18
  151. rasa/shared/providers/embedding/azure_openai_embedding_client.py +21 -54
  152. rasa/shared/providers/embedding/default_litellm_embedding_client.py +0 -24
  153. rasa/shared/providers/llm/_base_litellm_client.py +31 -63
  154. rasa/shared/providers/llm/azure_openai_llm_client.py +29 -50
  155. rasa/shared/providers/llm/default_litellm_llm_client.py +0 -24
  156. rasa/shared/providers/llm/self_hosted_llm_client.py +29 -17
  157. rasa/shared/providers/mappings.py +0 -19
  158. rasa/shared/utils/common.py +2 -37
  159. rasa/shared/utils/io.py +6 -28
  160. rasa/shared/utils/llm.py +46 -353
  161. rasa/shared/utils/yaml.py +82 -181
  162. rasa/studio/auth.py +5 -3
  163. rasa/studio/config.py +4 -13
  164. rasa/studio/constants.py +0 -1
  165. rasa/studio/data_handler.py +4 -13
  166. rasa/studio/upload.py +80 -175
  167. rasa/telemetry.py +17 -94
  168. rasa/tracing/config.py +1 -3
  169. rasa/tracing/instrumentation/attribute_extractors.py +17 -94
  170. rasa/tracing/instrumentation/instrumentation.py +0 -121
  171. rasa/utils/common.py +0 -5
  172. rasa/utils/endpoints.py +1 -27
  173. rasa/utils/io.py +81 -7
  174. rasa/utils/log_utils.py +2 -9
  175. rasa/utils/tensorflow/model_data.py +193 -2
  176. rasa/validator.py +4 -110
  177. rasa/version.py +1 -1
  178. rasa_pro-3.11.0a1.dist-info/METADATA +576 -0
  179. {rasa_pro-3.11.0.dist-info → rasa_pro-3.11.0a1.dist-info}/RECORD +182 -216
  180. rasa/core/actions/action_repeat_bot_messages.py +0 -89
  181. rasa/core/channels/inspector/dist/assets/flowDiagram-v2-855bc5b3-736177bf.js +0 -1
  182. rasa/core/channels/inspector/src/helpers/audiostream.ts +0 -165
  183. rasa/core/channels/voice_stream/asr/azure.py +0 -129
  184. rasa/core/channels/voice_stream/browser_audio.py +0 -107
  185. rasa/core/channels/voice_stream/call_state.py +0 -23
  186. rasa/dialogue_understanding/commands/repeat_bot_messages_command.py +0 -60
  187. rasa/dialogue_understanding/commands/user_silence_command.py +0 -59
  188. rasa/dialogue_understanding/patterns/repeat.py +0 -37
  189. rasa/dialogue_understanding/patterns/user_silence.py +0 -37
  190. rasa/model_manager/__init__.py +0 -0
  191. rasa/model_manager/config.py +0 -40
  192. rasa/model_manager/model_api.py +0 -559
  193. rasa/model_manager/runner_service.py +0 -286
  194. rasa/model_manager/socket_bridge.py +0 -146
  195. rasa/model_manager/studio_jwt_auth.py +0 -86
  196. rasa/model_manager/trainer_service.py +0 -325
  197. rasa/model_manager/utils.py +0 -87
  198. rasa/model_manager/warm_rasa_process.py +0 -187
  199. rasa/model_service.py +0 -112
  200. rasa/shared/core/flows/utils.py +0 -39
  201. rasa/shared/providers/_configs/litellm_router_client_config.py +0 -220
  202. rasa/shared/providers/_configs/model_group_config.py +0 -167
  203. rasa/shared/providers/_configs/rasa_llm_client_config.py +0 -73
  204. rasa/shared/providers/_utils.py +0 -79
  205. rasa/shared/providers/embedding/litellm_router_embedding_client.py +0 -135
  206. rasa/shared/providers/llm/litellm_router_llm_client.py +0 -182
  207. rasa/shared/providers/llm/rasa_llm_client.py +0 -112
  208. rasa/shared/providers/router/__init__.py +0 -0
  209. rasa/shared/providers/router/_base_litellm_router_client.py +0 -183
  210. rasa/shared/providers/router/router_client.py +0 -73
  211. rasa/shared/utils/health_check/__init__.py +0 -0
  212. rasa/shared/utils/health_check/embeddings_health_check_mixin.py +0 -31
  213. rasa/shared/utils/health_check/health_check.py +0 -258
  214. rasa/shared/utils/health_check/llm_health_check_mixin.py +0 -31
  215. rasa/utils/sanic_error_handler.py +0 -32
  216. rasa/utils/tensorflow/feature_array.py +0 -366
  217. rasa_pro-3.11.0.dist-info/METADATA +0 -198
  218. {rasa_pro-3.11.0.dist-info → rasa_pro-3.11.0a1.dist-info}/NOTICE +0 -0
  219. {rasa_pro-3.11.0.dist-info → rasa_pro-3.11.0a1.dist-info}/WHEEL +0 -0
  220. {rasa_pro-3.11.0.dist-info → rasa_pro-3.11.0a1.dist-info}/entry_points.txt +0 -0
@@ -1,89 +0,0 @@
1
- from typing import Optional, Dict, Any, List
2
-
3
- from rasa.core.actions.action import Action
4
- from rasa.core.channels import OutputChannel
5
- from rasa.core.nlg import NaturalLanguageGenerator
6
- from rasa.dialogue_understanding.patterns.collect_information import (
7
- CollectInformationPatternFlowStackFrame,
8
- )
9
- from rasa.dialogue_understanding.patterns.repeat import (
10
- RepeatBotMessagesPatternFlowStackFrame,
11
- )
12
- from rasa.dialogue_understanding.patterns.user_silence import (
13
- UserSilencePatternFlowStackFrame,
14
- )
15
- from rasa.shared.core.constants import ACTION_REPEAT_BOT_MESSAGES
16
- from rasa.shared.core.domain import Domain
17
- from rasa.shared.core.events import Event, BotUttered, UserUttered
18
- from rasa.shared.core.trackers import DialogueStateTracker
19
-
20
-
21
- class ActionRepeatBotMessages(Action):
22
- """Action to repeat bot messages"""
23
-
24
- def name(self) -> str:
25
- """Return the name of the action."""
26
- return ACTION_REPEAT_BOT_MESSAGES
27
-
28
- def _get_last_bot_events(self, tracker: DialogueStateTracker) -> List[Event]:
29
- """Get the last consecutive bot events before the most recent user message.
30
-
31
- This function scans the dialogue history in reverse to find the last sequence of
32
- bot responses that occurred without any user interruption. It filters out all
33
- non-utterance events and stops when it encounters a user message after finding
34
- bot messages.
35
-
36
- Args:
37
- tracker: DialogueStateTracker containing the conversation events.
38
-
39
- Returns:
40
- List[Event]: A list of consecutive BotUttered events that occurred
41
- most recently, in chronological order. Returns an empty list
42
- if no bot messages are found or if the last message was from
43
- the user.
44
-
45
- Example:
46
- For events: [User1, Bot1, Bot2, User2, Bot4, Bot5, User3]
47
- Returns: [Bot4, Bot5] (the last two bot events)
48
- The elif condition doesn't break when it sees User3 event.
49
- But it does at User2 event.
50
- """
51
- # Skip action if we are in a collect information step whose
52
- # default behavior is to repeat anyways
53
- top_frame = tracker.stack.top(
54
- lambda frame: isinstance(frame, RepeatBotMessagesPatternFlowStackFrame)
55
- or isinstance(frame, UserSilencePatternFlowStackFrame)
56
- )
57
- if isinstance(top_frame, CollectInformationPatternFlowStackFrame):
58
- return []
59
- # filter user and bot events
60
- filtered = [
61
- e for e in tracker.events if isinstance(e, (BotUttered, UserUttered))
62
- ]
63
- bot_events: List[Event] = []
64
-
65
- # find the last BotUttered events
66
- for e in reversed(filtered):
67
- if isinstance(e, BotUttered):
68
- # insert instead of append because the list is reversed
69
- bot_events.insert(0, e)
70
-
71
- # stop if a UserUttered event is found
72
- # only if we have collected some bot events already
73
- # this condition skips the first N UserUttered events
74
- elif bot_events:
75
- break
76
-
77
- return bot_events
78
-
79
- async def run(
80
- self,
81
- output_channel: OutputChannel,
82
- nlg: NaturalLanguageGenerator,
83
- tracker: DialogueStateTracker,
84
- domain: Domain,
85
- metadata: Optional[Dict[str, Any]] = None,
86
- ) -> List[Event]:
87
- """Send the last bot messages to the channel again"""
88
- bot_events = self._get_last_bot_events(tracker)
89
- return bot_events
@@ -1 +0,0 @@
1
- import{p as e,f as o}from"./flowDb-1972c806-fca3bfe4.js";import{f as t,g as a}from"./styles-080da4f6-23ffa4fc.js";import{t as s}from"./index-e7cef9de.js";import"./layout-b9885fb6.js";import"./index-2c4b9a3b-f55afcdf.js";import"./edges-f2ad444c-6166330c.js";import"./createText-62fc7601-b0ec81d6.js";import"./line-7c59abb6.js";import"./array-9f3ba611.js";import"./path-53f90ab3.js";const k={parser:e,db:o,renderer:t,styles:a,init:r=>{r.flowchart||(r.flowchart={}),r.flowchart.arrowMarkerAbsolute=r.arrowMarkerAbsolute,s({flowchart:{arrowMarkerAbsolute:r.arrowMarkerAbsolute}}),t.setConf(r.flowchart),o.clear(),o.setGen("gen-2")}};export{k as diagram};
@@ -1,165 +0,0 @@
1
- const bufferSize = 4096
2
- const sampleRate = 8000
3
- const audioOptions = {
4
- audio: {
5
- echoCancellation: true,
6
- noiseSuppression: true,
7
- autoGainControl: true
8
- }
9
- }
10
-
11
- const arrayBufferToBase64 = ( buffer: ArrayBuffer ): string => {
12
- let binary = '';
13
- const bytes = new Uint8Array( buffer );
14
- const len = bytes.byteLength;
15
- for (let i = 0; i < len; i++) {
16
- binary += String.fromCharCode( bytes[ i ] );
17
- }
18
- return window.btoa( binary );
19
- }
20
-
21
- const base64ToArrayBuffer = ( s: string ): ArrayBuffer => {
22
- const binary_string = window.atob(s);
23
- const len = binary_string.length;
24
- const bytes = new Uint8Array( len );
25
- for (let i = 0; i < len; i++) {
26
- bytes[i] = binary_string.charCodeAt(i);
27
- }
28
- return bytes.buffer;
29
- }
30
-
31
- const floatToIntArray = (arr: Float32Array): Int32Array => {
32
- // Convert Float Array [-1, 1] to full range int array
33
- return Int32Array.from(arr, x => x * 0x7fffffff)
34
- }
35
-
36
- const intToFloatArray = (arr: Int32Array): Float32Array => {
37
- return Float32Array.from(arr, x => (x / 0x7fffffff))
38
- }
39
-
40
- interface Mark {
41
- id: string
42
- bytesToGo: number
43
- }
44
-
45
- interface AudioQueue {
46
- buffer: Float32Array;
47
- marks: Array<Mark>
48
- socket: WebSocket,
49
- write: (newAudio: Float32Array) => void;
50
- read: (nSamples: number) => Float32Array;
51
- length: () => number;
52
- addMarker: (id: string) => void;
53
- reduceMarkers: (bytesRead: number) => void;
54
- popMarkers: () => void;
55
- }
56
-
57
-
58
- const createAudioQueue = (socket: WebSocket) : AudioQueue => {
59
- return {
60
- buffer: new Float32Array(0),
61
- marks: new Array<Mark>(),
62
- socket,
63
-
64
- write: function(newAudio: Float32Array) {
65
- const currentQLength = this.buffer.length;
66
- const newBuffer = new Float32Array(currentQLength + newAudio.length);
67
- newBuffer.set(this.buffer, 0);
68
- newBuffer.set(newAudio, currentQLength);
69
- this.buffer = newBuffer;
70
- },
71
-
72
- read: function(nSamples: number) {
73
- const samplesToPlay = this.buffer.subarray(0, nSamples);
74
- this.buffer = this.buffer.subarray(nSamples, this.buffer.length);
75
- this.reduceMarkers(samplesToPlay.length)
76
- this.popMarkers()
77
- return samplesToPlay;
78
- },
79
-
80
- length: function() {
81
- return this.buffer.length;
82
- },
83
-
84
- addMarker: function(id: string) {
85
- this.marks.push({id, bytesToGo: this.length()})
86
- },
87
-
88
- reduceMarkers: function(bytesRead: number) {
89
- this.marks = this.marks.map((m) => {
90
- return {id: m.id, bytesToGo: m.bytesToGo - bytesRead}
91
- })
92
- },
93
-
94
- popMarkers: function() {
95
- // marks are ordered
96
- let popUpTo = 0;
97
- while (popUpTo < this.marks.length) {
98
- if (this.marks[popUpTo].bytesToGo <= 0) {
99
- popUpTo += 1
100
- } else {
101
- break
102
- }
103
- }
104
- const marksToPop = this.marks.slice(0, popUpTo)
105
- this.marks = this.marks.slice(popUpTo, this.marks.length)
106
- marksToPop.forEach((m) => {
107
- this.socket.send(JSON.stringify({marker: m.id}))
108
- })
109
- }
110
-
111
- };
112
- }
113
-
114
- const streamMicrophoneToServer = async (socket: WebSocket) => {
115
- let audioStream = null;
116
- const audioContext = new AudioContext({sampleRate});
117
-
118
- try {
119
- audioStream = await navigator.mediaDevices.getUserMedia(audioOptions);
120
- const audioInput = audioContext.createMediaStreamSource(audioStream)
121
- const sender = audioContext.createScriptProcessor(bufferSize, 1, 1)
122
- sender.onaudioprocess = function(event) {
123
- const message = JSON.stringify({
124
- "audio": arrayBufferToBase64(floatToIntArray(event.inputBuffer.getChannelData(0)).buffer)
125
- })
126
- socket.send(message)
127
- }
128
- audioInput.connect(sender)
129
- sender.connect(audioContext.destination)
130
- } catch (err) {
131
- console.error(err);
132
- }
133
- }
134
-
135
- const setupAudioPlayback = (socket: WebSocket): AudioQueue => {
136
- const audioQueue = createAudioQueue(socket)
137
- const silence = new Float32Array(bufferSize)
138
- const audioOutputContext = new AudioContext({sampleRate})
139
- const scriptNode = audioOutputContext.createScriptProcessor(bufferSize, 1, 1);
140
- scriptNode.onaudioprocess = function(e) {
141
- const audioData = audioQueue.length() ? audioQueue.read(bufferSize) : silence
142
- e.outputBuffer.getChannelData(0).set(audioData);
143
- }
144
- scriptNode.connect(audioOutputContext.destination)
145
- return audioQueue
146
- }
147
-
148
- const addDataToAudioQueue = (audioQueue: AudioQueue) => (message: MessageEvent<any>) => {
149
- const data = JSON.parse(message.data.toString())
150
- if (data["audio"]) {
151
- const audioBytes = base64ToArrayBuffer(data["audio"])
152
- const audioData = intToFloatArray(new Int32Array(audioBytes))
153
- audioQueue.write(audioData);
154
- } else if (data["marker"]) {
155
- audioQueue.addMarker(data["marker"])
156
- }
157
- }
158
-
159
- export async function createAudioConnection() {
160
- const websocketURL = "ws://localhost:5005/webhooks/browser_audio/websocket"
161
- const socket = new WebSocket(websocketURL)
162
- socket.onopen = async () => { await streamMicrophoneToServer(socket)}
163
- const audioQueue = setupAudioPlayback(socket)
164
- socket.onmessage = addDataToAudioQueue(audioQueue)
165
- }
@@ -1,129 +0,0 @@
1
- import os
2
- from dataclasses import dataclass
3
- from typing import Any, Dict, Optional, AsyncIterator
4
- import asyncio
5
-
6
- from rasa.core.channels.voice_stream.asr.asr_engine import ASREngine, ASREngineConfig
7
- from rasa.core.channels.voice_stream.asr.asr_event import (
8
- ASREvent,
9
- NewTranscript,
10
- UserIsSpeaking,
11
- )
12
- from rasa.core.channels.voice_stream.audio_bytes import HERTZ, RasaAudioBytes
13
- from rasa.shared.constants import AZURE_SPEECH_API_KEY_ENV_VAR
14
- from rasa.shared.exceptions import ConnectionException
15
-
16
-
17
- @dataclass
18
- class AzureASRConfig(ASREngineConfig):
19
- language: Optional[str] = None
20
- speech_region: Optional[str] = None
21
-
22
-
23
- class AzureASR(ASREngine[AzureASRConfig]):
24
- required_env_vars = (AZURE_SPEECH_API_KEY_ENV_VAR,)
25
- required_packages = ("azure.cognitiveservices.speech",)
26
-
27
- def __init__(self, config: Optional[AzureASRConfig] = None):
28
- super().__init__(config)
29
-
30
- import azure.cognitiveservices.speech as speechsdk
31
-
32
- self.speech_recognizer: Optional[speechsdk.SpeechRecognizer] = None
33
- self.stream: Optional[speechsdk.audio.PushAudioInputStream] = None
34
- self.is_recognizing = False
35
- self.queue: asyncio.Queue[speechsdk.SpeechRecognitionEventArgs] = (
36
- asyncio.Queue()
37
- )
38
-
39
- @staticmethod
40
- def validate_environment() -> None:
41
- """Make sure all needed requirements for this component are met."""
42
-
43
- def signal_user_is_speaking(self, event: Any) -> None:
44
- """Replace the azure event with a generic is speaking event."""
45
- self.fill_queue(UserIsSpeaking())
46
-
47
- def fill_queue(self, event: Any) -> None:
48
- """Either puts the event or a dedicated ASR Event into the queue."""
49
- self.queue.put_nowait(event)
50
-
51
- async def connect(self) -> None:
52
- import azure.cognitiveservices.speech as speechsdk
53
-
54
- speech_config = speechsdk.SpeechConfig(
55
- subscription=os.environ[AZURE_SPEECH_API_KEY_ENV_VAR],
56
- region=self.config.speech_region,
57
- )
58
- audio_format = speechsdk.audio.AudioStreamFormat(
59
- samples_per_second=HERTZ,
60
- bits_per_sample=8,
61
- channels=1,
62
- wave_stream_format=speechsdk.AudioStreamWaveFormat.MULAW,
63
- )
64
- self.stream = speechsdk.audio.PushAudioInputStream(stream_format=audio_format)
65
- audio_config = speechsdk.audio.AudioConfig(stream=self.stream)
66
- self.speech_recognizer = speechsdk.SpeechRecognizer(
67
- speech_config=speech_config,
68
- language=self.config.language,
69
- audio_config=audio_config,
70
- )
71
- self.speech_recognizer.recognized.connect(self.fill_queue)
72
- self.speech_recognizer.recognizing.connect(self.signal_user_is_speaking)
73
- self.speech_recognizer.start_continuous_recognition_async()
74
- self.is_recognizing = True
75
-
76
- async def close_connection(self) -> None:
77
- if self.speech_recognizer is None:
78
- raise ConnectionException("Websocket not connected.")
79
- self.speech_recognizer.stop_continuous_recognition_async()
80
-
81
- async def signal_audio_done(self) -> None:
82
- """Signal to the ASR Api that you are done sending data."""
83
- self.is_recognizing = False
84
-
85
- def rasa_audio_bytes_to_engine_bytes(self, chunk: RasaAudioBytes) -> bytes:
86
- """Convert RasaAudioBytes to bytes usable by this engine."""
87
- return chunk
88
-
89
- async def send_audio_chunks(self, chunk: RasaAudioBytes) -> None:
90
- """Send audio chunks to the ASR system via the websocket."""
91
- if self.speech_recognizer is None or self.stream is None:
92
- raise ConnectionException("ASR not connected.")
93
- engine_bytes = self.rasa_audio_bytes_to_engine_bytes(chunk)
94
- self.stream.write(engine_bytes)
95
-
96
- async def stream_asr_events(self) -> AsyncIterator[ASREvent]:
97
- """Stream the events returned by the ASR system as it is fed audio bytes."""
98
- if self.speech_recognizer is None:
99
- raise ConnectionException("Websocket not connected.")
100
- while self.is_recognizing or not self.queue.empty():
101
- try:
102
- message = await asyncio.wait_for(self.queue.get(), timeout=2)
103
- asr_event = self.engine_event_to_asr_event(message)
104
- if asr_event:
105
- yield asr_event
106
- except asyncio.TimeoutError:
107
- pass
108
-
109
- def engine_event_to_asr_event(self, e: Any) -> Optional[ASREvent]:
110
- """Translate an engine event to a common ASREvent."""
111
- import azure.cognitiveservices.speech as speechsdk
112
-
113
- if isinstance(e, speechsdk.SpeechRecognitionEventArgs) and isinstance(
114
- e.result, speechsdk.SpeechRecognitionResult
115
- ):
116
- return NewTranscript(e.result.text)
117
- if isinstance(e, ASREvent):
118
- # transformation happened before
119
- return e
120
-
121
- return None
122
-
123
- @staticmethod
124
- def get_default_config() -> AzureASRConfig:
125
- return AzureASRConfig("en-US", "germanywestcentral")
126
-
127
- @classmethod
128
- def from_config_dict(cls, config: Dict) -> "AzureASR":
129
- return AzureASR(AzureASRConfig.from_dict(config))
@@ -1,107 +0,0 @@
1
- import audioop
2
- import base64
3
- import json
4
-
5
- import structlog
6
- import uuid
7
- from typing import Any, Awaitable, Callable, Optional, Tuple
8
-
9
- from sanic import Blueprint, HTTPResponse, Request, response
10
- from sanic import Websocket # type: ignore
11
-
12
-
13
- from rasa.core.channels import UserMessage
14
- from rasa.core.channels.voice_ready.utils import CallParameters
15
- from rasa.core.channels.voice_stream.call_state import call_state
16
- from rasa.core.channels.voice_stream.tts.tts_engine import TTSEngine
17
- from rasa.core.channels.voice_stream.audio_bytes import RasaAudioBytes
18
- from rasa.core.channels.voice_stream.voice_channel import (
19
- ContinueConversationAction,
20
- EndConversationAction,
21
- NewAudioAction,
22
- VoiceChannelAction,
23
- VoiceInputChannel,
24
- VoiceOutputChannel,
25
- )
26
-
27
- logger = structlog.get_logger()
28
-
29
-
30
- class BrowserAudioOutputChannel(VoiceOutputChannel):
31
- @classmethod
32
- def name(cls) -> str:
33
- return "browser_audio"
34
-
35
- def rasa_audio_bytes_to_channel_bytes(
36
- self, rasa_audio_bytes: RasaAudioBytes
37
- ) -> bytes:
38
- return audioop.ulaw2lin(rasa_audio_bytes, 4)
39
-
40
- def channel_bytes_to_message(self, recipient_id: str, channel_bytes: bytes) -> str:
41
- return json.dumps({"audio": base64.b64encode(channel_bytes).decode("utf-8")})
42
-
43
- def create_marker_message(self, recipient_id: str) -> Tuple[str, str]:
44
- message_id = uuid.uuid4().hex
45
- return json.dumps({"marker": message_id}), message_id
46
-
47
-
48
- class BrowserAudioInputChannel(VoiceInputChannel):
49
- @classmethod
50
- def name(cls) -> str:
51
- return "browser_audio"
52
-
53
- def channel_bytes_to_rasa_audio_bytes(self, input_bytes: bytes) -> RasaAudioBytes:
54
- return RasaAudioBytes(audioop.lin2ulaw(input_bytes, 4))
55
-
56
- async def collect_call_parameters(
57
- self, channel_websocket: Websocket
58
- ) -> Optional[CallParameters]:
59
- call_id = f"inspect-{uuid.uuid4()}"
60
- return CallParameters(call_id, "local", "local", stream_id=call_id)
61
-
62
- def map_input_message(
63
- self,
64
- message: Any,
65
- ) -> VoiceChannelAction:
66
- data = json.loads(message)
67
- if "audio" in data:
68
- channel_bytes = base64.b64decode(data["audio"])
69
- audio_bytes = self.channel_bytes_to_rasa_audio_bytes(channel_bytes)
70
- return NewAudioAction(audio_bytes)
71
- elif "marker" in data:
72
- if data["marker"] == call_state.latest_bot_audio_id:
73
- # Just finished streaming last audio bytes
74
- call_state.is_bot_speaking = False # type: ignore[attr-defined]
75
- if call_state.should_hangup:
76
- logger.debug(
77
- "browser_audio.hangup", marker=call_state.latest_bot_audio_id
78
- )
79
- return EndConversationAction()
80
- else:
81
- call_state.is_bot_speaking = True # type: ignore[attr-defined]
82
- return ContinueConversationAction()
83
-
84
- def create_output_channel(
85
- self, voice_websocket: Websocket, tts_engine: TTSEngine
86
- ) -> VoiceOutputChannel:
87
- return BrowserAudioOutputChannel(
88
- voice_websocket,
89
- tts_engine,
90
- self.tts_cache,
91
- )
92
-
93
- def blueprint(
94
- self, on_new_message: Callable[[UserMessage], Awaitable[Any]]
95
- ) -> Blueprint:
96
- """Defines a Sanic bluelogger.debug."""
97
- blueprint = Blueprint("browser_audio", __name__)
98
-
99
- @blueprint.route("/", methods=["GET"])
100
- async def health(_: Request) -> HTTPResponse:
101
- return response.json({"status": "ok"})
102
-
103
- @blueprint.websocket("/websocket") # type: ignore
104
- async def handle_message(request: Request, ws: Websocket) -> None:
105
- await self.run_audio_streaming(on_new_message, ws)
106
-
107
- return blueprint
@@ -1,23 +0,0 @@
1
- import asyncio
2
- from contextvars import ContextVar
3
- from werkzeug.local import LocalProxy
4
- from dataclasses import dataclass
5
- from typing import Optional
6
-
7
-
8
- # Per voice session data
9
- # This is similar to how flask makes the "request" object available as a global variable
10
- # It's a "global" variable that is local to an async task (i.e. websocket session)
11
- @dataclass
12
- class CallState:
13
- is_user_speaking: bool = False
14
- is_bot_speaking: bool = False
15
- silence_timeout_watcher: Optional[asyncio.Task] = None
16
- silence_timeout: Optional[float] = None
17
- latest_bot_audio_id: Optional[str] = None
18
- should_hangup: bool = False
19
- connection_failed: bool = False
20
-
21
-
22
- _call_state: ContextVar[CallState] = ContextVar("call_state")
23
- call_state = LocalProxy(_call_state)
@@ -1,60 +0,0 @@
1
- from __future__ import annotations
2
-
3
- from dataclasses import dataclass
4
- from typing import Any, Dict, List
5
- from rasa.dialogue_understanding.commands import Command
6
- from rasa.dialogue_understanding.patterns.repeat import (
7
- RepeatBotMessagesPatternFlowStackFrame,
8
- )
9
- from rasa.shared.core.events import Event
10
- from rasa.shared.core.flows import FlowsList
11
- from rasa.shared.core.trackers import DialogueStateTracker
12
-
13
-
14
- @dataclass
15
- class RepeatBotMessagesCommand(Command):
16
- """A command to indicate that the bot should repeat its last messages."""
17
-
18
- @classmethod
19
- def command(cls) -> str:
20
- """Returns the command type."""
21
- return "repeat"
22
-
23
- @classmethod
24
- def from_dict(cls, data: Dict[str, Any]) -> RepeatBotMessagesCommand:
25
- """Converts the dictionary to a command.
26
-
27
- Returns:
28
- The converted dictionary.
29
- """
30
- return RepeatBotMessagesCommand()
31
-
32
- def run_command_on_tracker(
33
- self,
34
- tracker: DialogueStateTracker,
35
- all_flows: FlowsList,
36
- original_tracker: DialogueStateTracker,
37
- ) -> List[Event]:
38
- """Runs the command on the tracker.
39
- Get all the bot utterances until last user utterance and repeat them.
40
-
41
- Args:
42
- tracker: The tracker to run the command on.
43
- all_flows: All flows in the assistant.
44
- original_tracker: The tracker before any command was executed.
45
-
46
- Returns:
47
- The events to apply to the tracker.
48
- """
49
- stack = tracker.stack
50
- stack.push(RepeatBotMessagesPatternFlowStackFrame())
51
- return tracker.create_stack_updated_events(stack)
52
-
53
- def __hash__(self) -> int:
54
- return hash(self.command())
55
-
56
- def __eq__(self, other: object) -> bool:
57
- if not isinstance(other, RepeatBotMessagesCommand):
58
- return False
59
-
60
- return True
@@ -1,59 +0,0 @@
1
- from __future__ import annotations
2
-
3
- from dataclasses import dataclass
4
- from typing import Any, Dict, List
5
- from rasa.dialogue_understanding.commands import Command
6
- from rasa.dialogue_understanding.patterns.user_silence import (
7
- UserSilencePatternFlowStackFrame,
8
- )
9
- from rasa.shared.core.events import Event
10
- from rasa.shared.core.flows import FlowsList
11
- from rasa.shared.core.trackers import DialogueStateTracker
12
-
13
-
14
- @dataclass
15
- class UserSilenceCommand(Command):
16
- """A command to indicate user silence."""
17
-
18
- @classmethod
19
- def command(cls) -> str:
20
- """Returns the command type."""
21
- return "user silence"
22
-
23
- @classmethod
24
- def from_dict(cls, data: Dict[str, Any]) -> UserSilenceCommand:
25
- """Converts the dictionary to a command.
26
-
27
- Returns:
28
- The converted dictionary.
29
- """
30
- return UserSilenceCommand()
31
-
32
- def run_command_on_tracker(
33
- self,
34
- tracker: DialogueStateTracker,
35
- all_flows: FlowsList,
36
- original_tracker: DialogueStateTracker,
37
- ) -> List[Event]:
38
- """Runs the command on the tracker.
39
-
40
- Args:
41
- tracker: The tracker to run the command on.
42
- all_flows: All flows in the assistant.
43
- original_tracker: The tracker before any command was executed.
44
-
45
- Returns:
46
- The events to apply to the tracker.
47
- """
48
- stack = tracker.stack
49
- stack.push(UserSilencePatternFlowStackFrame())
50
- return tracker.create_stack_updated_events(stack)
51
-
52
- def __hash__(self) -> int:
53
- return hash(self.command())
54
-
55
- def __eq__(self, other: object) -> bool:
56
- if not isinstance(other, UserSilenceCommand):
57
- return False
58
-
59
- return True
@@ -1,37 +0,0 @@
1
- from __future__ import annotations
2
-
3
- from dataclasses import dataclass
4
- from typing import Any, Dict
5
-
6
- from rasa.dialogue_understanding.stack.frames import PatternFlowStackFrame
7
- from rasa.shared.constants import RASA_DEFAULT_FLOW_PATTERN_PREFIX
8
-
9
- FLOW_PATTERN_REPEAT = RASA_DEFAULT_FLOW_PATTERN_PREFIX + "repeat_bot_messages"
10
-
11
-
12
- @dataclass
13
- class RepeatBotMessagesPatternFlowStackFrame(PatternFlowStackFrame):
14
- """A flow stack frame that can get added when bot messages should be repeated"""
15
-
16
- flow_id: str = FLOW_PATTERN_REPEAT
17
- """The ID of the flow."""
18
-
19
- @classmethod
20
- def type(cls) -> str:
21
- """Returns the type of the frame."""
22
- return FLOW_PATTERN_REPEAT
23
-
24
- @staticmethod
25
- def from_dict(data: Dict[str, Any]) -> RepeatBotMessagesPatternFlowStackFrame:
26
- """Creates a `DialogueStackFrame` from a dictionary.
27
-
28
- Args:
29
- data: The dictionary to create the `DialogueStackFrame` from.
30
-
31
- Returns:
32
- The created `DialogueStackFrame`.
33
- """
34
- return RepeatBotMessagesPatternFlowStackFrame(
35
- frame_id=data["frame_id"],
36
- step_id=data["step_id"],
37
- )