rasa-pro 3.13.0.dev5__py3-none-any.whl → 3.13.0.dev8__py3-none-any.whl

rasa/core/channels/inspector/src/components/Slots.tsx

@@ -11,24 +11,24 @@ import {
   Flex,
   useColorModeValue,
   Tooltip,
-} from "@chakra-ui/react";
-import { useOurTheme } from "../theme";
-import { Light as SyntaxHighlighter } from "react-syntax-highlighter";
-import { Slot } from "../types";
-import { shouldShowTooltip } from "../helpers/utils";
+} from '@chakra-ui/react'
+import { useOurTheme } from '../theme'
+import { Light as SyntaxHighlighter } from 'react-syntax-highlighter'
+import { Slot } from '../types'
+import { shouldShowTooltip } from '../helpers/utils'
 
 interface Props extends FlexProps {
-  slots: Slot[];
+  slots: Slot[]
 }
 
 export const SlotTable = ({ slots }: { slots: Slot[] }) => {
-  const { rasaFontSizes } = useOurTheme();
+  const { rasaFontSizes } = useOurTheme()
 
   const highlighterSx = {
-    background: useColorModeValue("neutral.50", "neutral.50"),
+    background: useColorModeValue('neutral.50', 'neutral.50'),
     fontSize: rasaFontSizes.sm,
-    letterSpacing: "0",
-  };
+    letterSpacing: '0',
+  }
 
   return (
     <Table width="100%" layout="fixed">
@@ -59,27 +59,27 @@ export const SlotTable = ({ slots }: { slots: Slot[] }) => {
         ))}
       </Tbody>
     </Table>
-  );
-};
+  )
+}
 
 export const Slots = ({ sx, slots, ...props }: Props) => {
-  const { rasaSpace } = useOurTheme();
+  const { rasaSpace } = useOurTheme()
 
   const containerSx = {
     ...sx,
     pr: 0,
     pb: 0,
-    position: "relative",
-    flexDirection: "column",
-  };
+    position: 'relative',
+    flexDirection: 'column',
+  }
   const overflowBox = {
-    height: "100%",
-    overflow: "auto",
+    height: '100%',
+    overflow: 'auto',
     pr: rasaSpace[1],
     pb: rasaSpace[0.5],
-  };
+  }
 
-  const displaySlots = slots.length ? slots : [{ name: "-", value: "-" }];
+  const displaySlots = slots.length ? slots : [{ name: '-', value: '-' }]
 
   return (
     <Flex sx={containerSx} {...props}>
@@ -87,5 +87,5 @@ export const Slots = ({ sx, slots, ...props }: Props) => {
         <SlotTable slots={displaySlots} />
       </Box>
     </Flex>
-  );
-};
+  )
+}

rasa/core/channels/inspector/src/components/Welcome.tsx

@@ -1,38 +1,35 @@
-import {
-  Box,
-  Flex,
-  FlexProps,
-  Heading,
-  Link,
-  Text,
-} from "@chakra-ui/react";
-import { RasaLogo, RasaLogoDark } from "./RasaLogo";
-import { useOurTheme } from "../theme";
+import { Box, Flex, FlexProps, Heading, Link, Text } from '@chakra-ui/react'
+import { RasaLogo, RasaLogoDark } from './RasaLogo'
+import { useOurTheme } from '../theme'
 
 interface WelcomeProps extends FlexProps {
-  isRecruitmentVisible?: boolean;
+  isRecruitmentVisible?: boolean
 }
 
-export const Welcome = ({ sx, isRecruitmentVisible, ...props }: WelcomeProps) => {
-  const { rasaSpace } = useOurTheme();
+export const Welcome = ({
+  sx,
+  isRecruitmentVisible,
+  ...props
+}: WelcomeProps) => {
+  const { rasaSpace } = useOurTheme()
 
   const containerSx = {
     ...sx,
-    color: isRecruitmentVisible ? "black" : "neutral.50",
-    bg: isRecruitmentVisible ? "white" : undefined,
-    bgGradient: isRecruitmentVisible ? undefined : "linear(to-b, #4E61E1, #7622D2)",
-  };
+    color: isRecruitmentVisible ? 'black' : 'neutral.50',
+    bg: isRecruitmentVisible ? 'white' : undefined,
+    bgGradient: isRecruitmentVisible
+      ? undefined
+      : 'linear(to-b, #4E61E1, #7622D2)',
+  }
 
   const linkSx = {
     flexGrow: 0,
-    color: isRecruitmentVisible ? "#0000EE" : "neutral.50",
-    textDecoration: "underline",
+    color: isRecruitmentVisible ? '#0000EE' : 'neutral.50',
+    textDecoration: 'underline',
     _hover: {
-      color: isRecruitmentVisible 
-        ? "link.visited" 
-        : "neutral.400",
+      color: isRecruitmentVisible ? 'link.visited' : 'neutral.400',
     },
-  };
+  }
 
   return (
     <Flex sx={containerSx} {...props}>
@@ -40,21 +37,21 @@ export const Welcome = ({ sx, isRecruitmentVisible, ...props }: WelcomeProps) =>
         <Heading as="h1" size="xl" mb={rasaSpace[1]}>
           Rasa Inspector
         </Heading>
-        <Text as="span">
-          New to the Inspector?
-        </Text>
+        <Text as="span">New to the Inspector?</Text>
         <Link
           sx={linkSx}
           href="https://rasa.com/docs/rasa-pro/production/inspect-assistant/"
           target="_blank"
           ml={rasaSpace[0.25]}
-        >Browse the docs</Link>
+        >
+          Browse the docs
+        </Link>
       </Box>
       {isRecruitmentVisible ? (
-        <RasaLogoDark sx={{ flexShrink: 0, marginLeft: "auto" }}/>
+        <RasaLogoDark sx={{ flexShrink: 0, marginLeft: 'auto' }} />
       ) : (
-        <RasaLogo sx={{ flexShrink: 0, marginLeft: "auto" }}/>
+        <RasaLogo sx={{ flexShrink: 0, marginLeft: 'auto' }} />
       )}
     </Flex>
-  );
-};
+  )
+}

rasa/core/channels/inspector/src/helpers/audio/audiostream.ts

@@ -0,0 +1,245 @@
+const bufferSize = 128
+const sampleRate = 8000
+const audioOptions = {
+  audio: {
+    echoCancellation: true,
+    noiseSuppression: true,
+    autoGainControl: true,
+  },
+}
+
+const arrayBufferToBase64 = (buffer: ArrayBuffer): string => {
+  let binary = ''
+  const bytes = new Uint8Array(buffer)
+  const len = bytes.byteLength
+  for (let i = 0; i < len; i++) {
+    binary += String.fromCharCode(bytes[i])
+  }
+  return window.btoa(binary)
+}
+
+const base64ToArrayBuffer = (s: string): ArrayBuffer => {
+  const binary_string = window.atob(s)
+  const len = binary_string.length
+  const bytes = new Uint8Array(len)
+  for (let i = 0; i < len; i++) {
+    bytes[i] = binary_string.charCodeAt(i)
+  }
+  return bytes.buffer
+}
+
+const floatToIntArray = (arr: Float32Array): Int32Array => {
+  // Convert Float Array [-1, 1] to full range int array
+  return Int32Array.from(arr, (x) => x * 0x7fffffff)
+}
+
+const intToFloatArray = (arr: Int32Array): Float32Array => {
+  return Float32Array.from(arr, (x) => x / 0x7fffffff)
+}
+
+interface Mark {
+  id: string
+  bytesToGo: number
+}
+
+interface AudioQueue {
+  buffer: Float32Array
+  marks: Array<Mark>
+  socket: WebSocket
+  write: (newAudio: Float32Array) => void
+  read: (nSamples: number) => Float32Array
+  length: () => number
+  addMarker: (id: string) => void
+  reduceMarkers: (bytesRead: number) => void
+  popMarkers: () => void
+}
+
+const createAudioQueue = (socket: WebSocket): AudioQueue => {
+  return {
+    buffer: new Float32Array(0),
+    marks: new Array<Mark>(),
+    socket,
+
+    write: function (newAudio: Float32Array) {
+      const currentQLength = this.buffer.length
+      const newBuffer = new Float32Array(currentQLength + newAudio.length)
+      newBuffer.set(this.buffer, 0)
+      newBuffer.set(newAudio, currentQLength)
+      this.buffer = newBuffer
+    },
+
+    read: function (nSamples: number) {
+      const samplesToPlay = this.buffer.subarray(0, nSamples)
+      this.buffer = this.buffer.subarray(nSamples, this.buffer.length)
+      this.reduceMarkers(samplesToPlay.length)
+      this.popMarkers()
+      return samplesToPlay
+    },
+
+    length: function () {
+      return this.buffer.length
+    },
+
+    addMarker: function (id: string) {
+      this.marks.push({ id, bytesToGo: this.length() })
+    },
+
+    reduceMarkers: function (bytesRead: number) {
+      this.marks = this.marks.map((m) => {
+        return { id: m.id, bytesToGo: m.bytesToGo - bytesRead }
+      })
+    },
+
+    popMarkers: function () {
+      // marks are ordered
+      let popUpTo = 0
+      while (popUpTo < this.marks.length) {
+        if (this.marks[popUpTo].bytesToGo <= 0) {
+          popUpTo += 1
+        } else {
+          break
+        }
+      }
+      const marksToPop = this.marks.slice(0, popUpTo)
+      this.marks = this.marks.slice(popUpTo, this.marks.length)
+      marksToPop.forEach((m) => {
+        if (this.socket.readyState === WebSocket.OPEN) {
+          this.socket.send(JSON.stringify({ marker: m.id }))
+        }
+      })
+    },
+  }
+}
+
+const streamMicrophoneToServer = async (socket: WebSocket) => {
+  const audioContext = new AudioContext({ sampleRate })
+
+  try {
+    const audioStream = await navigator.mediaDevices.getUserMedia(audioOptions)
+    await audioContext.audioWorklet.addModule(
+      new URL('./microphone-processor.js', import.meta.url).href,
+    )
+
+    const microphoneNode = new AudioWorkletNode(
+      audioContext,
+      'microphone-processor',
+    )
+    microphoneNode.port.onmessage = (event: MessageEvent) => {
+      const audioData = event.data as Float32Array
+      const message = JSON.stringify({
+        audio: arrayBufferToBase64(floatToIntArray(audioData).buffer),
+      })
+      if (socket.readyState === WebSocket.OPEN) {
+        socket.send(message)
+      }
+    }
+    const source = audioContext.createMediaStreamSource(audioStream)
+    source.connect(microphoneNode).connect(audioContext.destination)
+  } catch (err) {
+    console.error(err)
+  }
+}
+
+const setupAudioPlayback = async (socket: WebSocket): Promise<AudioQueue> => {
+  const audioQueue = createAudioQueue(socket)
+  const audioOutputContext = new AudioContext({ sampleRate })
+
+  // Resume the audio context (browsers often start it in suspended state)
+  if (audioOutputContext.state === 'suspended') {
+    await audioOutputContext.resume()
+  }
+
+  await audioOutputContext.audioWorklet.addModule(
+    new URL('./playback-processor.js', import.meta.url).href,
+  )
+
+  const playbackNode = new AudioWorkletNode(
+    audioOutputContext,
+    'playback-processor',
+  )
+
+  playbackNode.port.onmessage = (event: MessageEvent) => {
+    if (event.data === 'need-more-data') {
+      // Request more audio data from the queue
+      const audioData = audioQueue.length()
+        ? audioQueue.read(bufferSize)
+        : new Float32Array(bufferSize)
+
+      if (!(audioData instanceof Float32Array)) {
+        console.error('audioData is invalid, sending silence.')
+        playbackNode.port.postMessage(new Float32Array(bufferSize))
+      } else if (audioData.length !== bufferSize) {
+        console.warn(
+          `audioData is too short (${audioData.length} samples), padding with silence`,
+        )
+        const padded = new Float32Array(bufferSize)
+        padded.set(audioData)
+        playbackNode.port.postMessage(padded)
+      } else {
+        playbackNode.port.postMessage(audioData)
+      }
+    }
+  }
+
+  playbackNode.connect(audioOutputContext.destination)
+
+  return audioQueue
+}
+
+const addDataToAudioQueue =
+  (audioQueue: AudioQueue) => (message: MessageEvent<any>) => {
+    try {
+      const data = JSON.parse(message.data.toString())
+      if (data['error']) {
+        console.error('Error from server:', data['error'])
+      }
+      if (data['audio']) {
+        const audioBytes = base64ToArrayBuffer(data['audio'])
+        const int32Data = new Int32Array(audioBytes)
+        const audioData = intToFloatArray(int32Data)
+        audioQueue.write(audioData)
+      } else if (data['marker']) {
+        audioQueue.addMarker(data['marker'])
+      }
+    } catch (error) {
+      console.error('Error processing server incoming audio data:', error)
+    }
+  }
+
+/**
+ * Constructs a WebSocket URL for browser audio from a base HTTP/HTTPS URL
+ *
+ * @param baseUrl - The base URL (e.g., "https://example.com" or "http://localhost:5005")
+ * @returns WebSocket URL for browser audio endpoint
+ *
+ * @example
+ * getWebSocketUrl("https://example.com")
+ * // Returns: "wss://example.com/webhooks/browser_audio/websocket"
+ *
+ * getWebSocketUrl("http://localhost:5005")
+ * // Returns: "ws://localhost:5005/webhooks/browser_audio/websocket"
+ *
+ * @throws {TypeError} If baseUrl is not a valid URL
+ */
+function getWebSocketUrl(baseUrl: string) {
+  const url = new URL(baseUrl)
+  const wsProtocol = url.protocol === 'https:' ? 'wss:' : 'ws:'
+  return `${wsProtocol}//${url.host}/webhooks/browser_audio/websocket`
+}
+
+/**
+ * Creates a WebSocket connection for browser audio and streams microphone input to the server
+ *
+ * @param baseUrl - The base URL (e.g., "https://example.com" or "http://localhost:5005")
+ */
+export async function createAudioConnection(baseUrl: string) {
+  const websocketURL = getWebSocketUrl(baseUrl)
+  const socket = new WebSocket(websocketURL)
+
+  socket.onopen = async () => {
+    await streamMicrophoneToServer(socket)
+  }
+
+  const audioQueue = await setupAudioPlayback(socket)
+  socket.onmessage = addDataToAudioQueue(audioQueue)
+}

rasa/core/channels/inspector/src/helpers/audio/microphone-processor.js

@@ -0,0 +1,12 @@
+class MicrophoneProcessor extends AudioWorkletProcessor {
+  process(inputs) {
+    const input = inputs[0]
+    if (input.length > 0) {
+      const channelData = input[0]
+      this.port.postMessage(channelData)
+    }
+    return true
+  }
+}
+
+registerProcessor('microphone-processor', MicrophoneProcessor)

rasa/core/channels/inspector/src/helpers/audio/playback-processor.js

@@ -0,0 +1,36 @@
+class PlaybackProcessor extends AudioWorkletProcessor {
+  constructor() {
+    super()
+    this.audioBuffer = new Float32Array(0)
+
+    // Set up message handling from the main thread
+    this.port.onmessage = (event) => {
+      this.audioBuffer = event.data
+    }
+
+    // Request initial audio data
+    this.port.postMessage('need-more-data')
+  }
+
+  process(_, outputs) {
+    const output = outputs[0]
+    const channelData = output[0]
+
+    const availableSamples = this.audioBuffer.length
+    const requestedSamples = channelData.length
+    const samplesToCopy = Math.min(availableSamples, requestedSamples)
+
+    if (samplesToCopy > 0) {
+      channelData.set(this.audioBuffer.subarray(0, samplesToCopy))
+      this.audioBuffer = this.audioBuffer.subarray(samplesToCopy)
+    } else {
+      channelData.fill(0)
+    }
+
+    this.port.postMessage('need-more-data')
+
+    return true
+  }
+}
+
+registerProcessor('playback-processor', PlaybackProcessor)

rasa/core/channels/inspector/src/helpers/conversation.ts

@@ -1,16 +1,16 @@
 export const restartConversation = () => {
   // unset the sender id from the query parameters
-  const url = new URL(window.location.href);
-  url.searchParams.delete("sender");
-  window.history.pushState(null, "", url.toString());
-  location.reload();
-};
+  const url = new URL(window.location.href)
+  url.searchParams.delete('sender')
+  window.history.pushState(null, '', url.toString())
+  location.reload()
+}
 
 // Make the function available on the window object
 declare global {
   interface Window {
-    restartConversation: typeof restartConversation;
+    restartConversation: typeof restartConversation
   }
 }
 
-window.restartConversation = restartConversation;
+window.restartConversation = restartConversation