rasa-pro 3.13.11__py3-none-any.whl → 3.14.0__py3-none-any.whl

rasa/core/channels/inspector/src/components/LatencyDisplay.tsx

@@ -0,0 +1,296 @@
+import {
+  Box,
+  Flex,
+  FlexProps,
+  Table,
+  Tbody,
+  Td,
+  Text,
+  Tooltip,
+  Tr,
+  useColorModeValue,
+} from '@chakra-ui/react'
+import { useOurTheme } from '../theme'
+import {
+  isRasaLatency,
+  isVoiceLatency,
+  RasaLatency,
+  VoiceLatency,
+} from '../types'
+
+interface MinimalDisplayProps extends FlexProps {
+  latency: RasaLatency
+}
+
+/**
+ * Simple latency display for text-only conversations.
+ * Shows a single response time value.
+ */
+const MinimalDisplay = ({ latency, sx, ...props }: MinimalDisplayProps) => {
+  const containerSx = {
+    ...sx,
+    display: 'flex',
+    alignItems: 'center',
+  }
+
+  const getLatencyColor = (latency: number) => {
+    if (latency < 1500) return 'green.500'
+    if (latency < 2500) return 'orange.500'
+    return 'red.500'
+  }
+
+  const value = Math.round(
+    (latency.command_processor || 0) + (latency.prediction_loop || 0),
+  )
+  const color = getLatencyColor(value)
+
+  return (
+    <Flex sx={containerSx} {...props}>
+      <Text fontSize="md" color={useColorModeValue('gray.700', 'gray.300')}>
+        Response latency:
+        <Text as="span" ml={2} fontWeight="bold" color={color}>
+          {value}ms
+        </Text>
+      </Text>
+    </Flex>
+  )
+}
+
+interface WaterfallDisplayProps extends FlexProps {
+  latency: VoiceLatency
+}
+
+/**
+ * Detailed latency waterfall chart for voice conversations.
+ * Displays processing times for ASR, Rasa, and TTS components.
+ */
+const WaterfallDisplay = ({ latency, sx, ...props }: WaterfallDisplayProps) => {
+  const { rasaSpace } = useOurTheme()
+
+  const containerSx = {
+    ...sx,
+    flexDirection: 'column',
+    gap: rasaSpace[1],
+  }
+
+  const headerSx = {
+    fontSize: 'sm',
+    fontWeight: 'bold',
+    color: useColorModeValue('gray.700', 'gray.300'),
+  }
+
+  const waterfallBarSx = {
+    height: '24px',
+    borderRadius: '4px',
+    overflow: 'hidden',
+    border: '1px solid',
+    borderColor: useColorModeValue('gray.200', 'gray.600'),
+  }
+
+  const legendTableSx = {
+    size: 'sm',
+    mt: rasaSpace[0.5],
+  }
+
+  const getLatencyColor = (type: string) => {
+    const colors: { [key: string]: string } = {
+      asr: 'blue.500',
+      rasa: 'purple.500',
+      tts_first: 'orange.500',
+      tts_complete: 'green.500',
+    }
+    return colors[type] || 'gray.500'
+  }
+
+  const getLatencyDescription = (type: string) => {
+    const descriptions: { [key: string]: string } = {
+      asr: 'Time from the first Partial Transcript event to the Final Transcript event from Speech Recognition. It also includes the time taken by the user to speak.',
+      rasa: 'Time taken by Rasa to process the text from the Final Transcript event from Speech Recognition until a text response is generated.',
+      tts_first:
+        'Time between the request sent to Text-to-Speech processing and the first byte of audio received by Rasa.',
+      tts_complete:
+        'Time taken by Text-to-Speech to complete audio generation. It depends on the length of the text and could overlap with the Bot speaking time.',
+    }
+    return descriptions[type] || ''
+  }
+
+  // Metrics for proportional display (only actual processing latencies)
+  const chartMetrics = [
+    latency.asr_latency_ms && {
+      type: 'asr',
+      label: 'ASR',
+      value: latency.asr_latency_ms,
+    },
+    latency.rasa_processing_latency_ms && {
+      type: 'rasa',
+      label: 'Rasa',
+      value: latency.rasa_processing_latency_ms,
+    },
+    latency.tts_complete_latency_ms && {
+      type: 'tts_complete',
+      label: 'TTS',
+      value: latency.tts_complete_latency_ms,
+    },
+  ].filter(Boolean)
+
+  // All metrics for legend display
+  const allMetrics = [
+    latency.asr_latency_ms && {
+      type: 'asr',
+      label: 'ASR',
+      value: latency.asr_latency_ms,
+    },
+    latency.rasa_processing_latency_ms && {
+      type: 'rasa',
+      label: 'Rasa',
+      value: latency.rasa_processing_latency_ms,
+    },
+    latency.tts_first_byte_latency_ms && {
+      type: 'tts_first',
+      label: 'TTS First Byte',
+      value: latency.tts_first_byte_latency_ms,
+    },
+    latency.tts_complete_latency_ms && {
+      type: 'tts_complete',
+      label: 'TTS Complete',
+      value: latency.tts_complete_latency_ms,
+    },
+  ].filter(Boolean)
+
+  // Calculate total for proportional sizing (only processing latencies)
+  const totalLatency = chartMetrics.reduce(
+    (sum: number, metric: any) => sum + metric.value,
+    0,
+  )
+
+  // Calculate total latency for title (Rasa + TTS First Byte)
+  const totalDisplayLatency =
+    (latency.rasa_processing_latency_ms || 0) +
+    (latency.tts_first_byte_latency_ms || 0)
+
+  return (
+    <Flex sx={containerSx} {...props}>
+      <Text sx={headerSx}>
+        Response latency: ~{Math.round(totalDisplayLatency)}ms
+      </Text>
+
+      {/* Waterfall Bar */}
+      <Box>
+        <Flex sx={waterfallBarSx}>
+          {chartMetrics.map((metric: any) => {
+            const widthPercentage =
+              totalLatency > 0 ? (metric.value / totalLatency) * 100 : 0
+            return (
+              <Tooltip
+                key={metric.type}
+                label={
+                  <Box>
+                    <Text fontWeight="bold">
+                      {metric.label}: {Math.round(metric.value)}ms
+                    </Text>
+                    <Text fontSize="xs" mt={1}>
+                      {getLatencyDescription(metric.type)}
+                    </Text>
+                  </Box>
+                }
+                hasArrow
+                placement="top"
+              >
+                <Box
+                  bg={getLatencyColor(metric.type)}
+                  width={`${widthPercentage}%`}
+                  height="100%"
+                  minWidth="40px" // Increased minimum width for better visibility
+                  cursor="pointer"
+                  _hover={{ opacity: 0.8 }}
+                />
+              </Tooltip>
+            )
+          })}
+        </Flex>
+
+        {/* Legend */}
+        <Table sx={legendTableSx}>
+          <Tbody>
+            {/* Split metrics into pairs for 2x2 table */}
+            {Array.from(
+              { length: Math.ceil(allMetrics.length / 2) },
+              (_, rowIndex) => (
+                <Tr key={rowIndex}>
+                  {allMetrics
+                    .slice(rowIndex * 2, rowIndex * 2 + 2)
+                    .map((metric: any) => (
+                      <Td key={metric.type} p={rasaSpace[0.25]} border="none">
+                        <Flex align="center" gap={rasaSpace[0.25]}>
+                          <Box
+                            width="8px"
+                            height="8px"
+                            bg={getLatencyColor(metric.type)}
+                            borderRadius="2px"
+                            flexShrink={0}
+                          />
+                          <Text
+                            fontSize="xs"
+                            color={useColorModeValue('gray.600', 'gray.400')}
+                            noOfLines={1}
+                          >
+                            {metric.label}: {Math.round(metric.value)}ms
+                          </Text>
+                        </Flex>
+                      </Td>
+                    ))}
+                  {/* Fill empty cell if odd number of metrics */}
+                  {allMetrics.length % 2 !== 0 &&
+                    rowIndex === Math.ceil(allMetrics.length / 2) - 1 && (
+                      <Td p={rasaSpace[0.25]} border="none" />
+                    )}
+                </Tr>
+              ),
+            )}
+          </Tbody>
+        </Table>
+      </Box>
+    </Flex>
+  )
+}
+
+interface LatencyDisplayProps extends FlexProps {
+  voiceLatency?: any
+  rasaLatency?: any
+}
+
+/**
+ * Displays processing latency information for the conversation.
+ * Shows either a detailed waterfall chart for voice conversations or
+ * a simpler display for text-only conversations.
+ */
+export const LatencyDisplay = ({
+  sx,
+  voiceLatency,
+  rasaLatency,
+  ...props
+}: LatencyDisplayProps) => {
+  // We give priority to voice latency as most comprehensive.
+  if (isVoiceLatency(voiceLatency)) {
+    return <WaterfallDisplay latency={voiceLatency} sx={sx} {...props} />
+  }
+
+  // If voice latency is not available, we use rasa latency.
+  if (isRasaLatency(rasaLatency)) {
+    return <MinimalDisplay latency={rasaLatency} sx={sx} {...props} />
+  }
+
+  // If rasa latency is not available, we are either waiting for first data from voice stream or the conversation is not started yet.
+  if (!rasaLatency) {
+    return <Text>Rasa latency data is not available yet</Text>
+  }
+
+  // If rasa latency is available but did not pass type guard, we are in an unexpected state
+  // and we should log an error but gracefully fallback to showing no data.
+  console.warn(
+    `Latency data has unexpected format. Raw data of rasaLatency: ${rasaLatency},
+    raw data of voice latency: ${voiceLatency}`,
+  )
+
+  return <Text>Latency data is not available yet</Text>
+}

rasa/core/channels/inspector/src/components/LoadingSpinner.tsx

@@ -8,7 +8,11 @@ import {
 import { useOurTheme } from '../theme'
 import { createAudioConnection } from '../helpers/audio/audiostream.ts'
 
-export const LoadingSpinner = () => {
+interface LoadingSpinnerProps {
+  onLatencyUpdate?: (latency: any) => void
+}
+
+export const LoadingSpinner = ({ onLatencyUpdate }: LoadingSpinnerProps) => {
   const { rasaSpace } = useOurTheme()
   const isVoice = window.location.href.includes('browser_audio')
   const text = isVoice
@@ -27,7 +31,7 @@ export const LoadingSpinner = () => {
       {isVoice ? (
         <Button
           onClick={async () =>
-            await createAudioConnection(window.location.href)
+            await createAudioConnection(window.location.href, onLatencyUpdate)
           }
         >
           Go

rasa/core/channels/inspector/src/helpers/audio/audiostream.ts

@@ -8,7 +8,7 @@ const audioOptions = {
   },
 }
 
-const arrayBufferToBase64 = (buffer: ArrayBuffer): string => {
+const arrayBufferToBase64 = (buffer: ArrayBufferLike): string => {
   let binary = ''
   const bytes = new Uint8Array(buffer)
   const len = bytes.byteLength
@@ -52,6 +52,7 @@ interface AudioQueue {
   addMarker: (id: string) => void
   reduceMarkers: (bytesRead: number) => void
   popMarkers: () => void
+  clear: () => void
 }
 
 const createAudioQueue = (socket: WebSocket): AudioQueue => {
@@ -108,6 +109,14 @@ const createAudioQueue = (socket: WebSocket): AudioQueue => {
         }
       })
     },
+
+    /**
+     * Clears the audio queue, removing all buffered audio and markers.
+     */
+    clear: function () {
+      this.buffer = new Float32Array(0)
+      this.marks = []
+    },
   }
 }
 
@@ -187,7 +196,8 @@ const setupAudioPlayback = async (socket: WebSocket): Promise<AudioQueue> => {
 }
 
 const addDataToAudioQueue =
-  (audioQueue: AudioQueue) => (message: MessageEvent<any>) => {
+  (audioQueue: AudioQueue, onLatencyUpdate?: (latency: any) => void) =>
+  (message: MessageEvent<any>) => {
     try {
       const data = JSON.parse(message.data.toString())
       if (data['error']) {
@@ -199,7 +209,15 @@ const addDataToAudioQueue =
         const audioData = intToFloatArray(int32Data)
         audioQueue.write(audioData)
       } else if (data['marker']) {
+        if (data['latency'] && onLatencyUpdate) {
+          onLatencyUpdate(data['latency'])
+        }
+        console.log('Voice Latency Metrics:', data['latency'])
         audioQueue.addMarker(data['marker'])
+      } else if (data['interruptPlayback']) {
+        // User interrupted the bot, immediately clear the audio queue
+        audioQueue.clear()
+        console.log('Audio queue cleared due to user interruption.')
       }
     } catch (error) {
       console.error('Error processing server incoming audio data:', error)
@@ -231,8 +249,12 @@ function getWebSocketUrl(baseUrl: string) {
  * Creates a WebSocket connection for browser audio and streams microphone input to the server
  *
  * @param baseUrl - The base URL (e.g., "https://example.com" or "http://localhost:5005")
+ * @param onLatencyUpdate - Optional callback function to receive latency updates
  */
-export async function createAudioConnection(baseUrl: string) {
+export async function createAudioConnection(
+  baseUrl: string,
+  onLatencyUpdate?: (latency: any) => void,
+) {
   const websocketURL = getWebSocketUrl(baseUrl)
   const socket = new WebSocket(websocketURL)
 
@@ -241,5 +263,5 @@ export async function createAudioConnection(baseUrl: string) {
   }
 
   const audioQueue = await setupAudioPlayback(socket)
-  socket.onmessage = addDataToAudioQueue(audioQueue)
+  socket.onmessage = addDataToAudioQueue(audioQueue, onLatencyUpdate)
 }

rasa/core/channels/inspector/src/helpers/formatters.test.ts

@@ -294,6 +294,7 @@ describe('helpers', () => {
           step_id: 'step_id',
           collect: fieldValue,
           ended: false,
+          type: 'flow',
         }),
       ).toEqual(fieldValue)
     })
@@ -307,6 +308,7 @@ describe('helpers', () => {
           flow_id: 'flow_id',
           step_id: 'step_id',
           ended: false,
+          type: 'flow',
         }),
       ).toEqual(fieldValue)
     })
@@ -321,6 +323,7 @@ describe('helpers', () => {
           step_id: 'step_id',
           collect: fieldValue,
           ended: false,
+          type: 'flow',
         }),
       ).toEqual(`${fieldValue} is not null`)
     })
@@ -335,6 +338,7 @@ describe('helpers', () => {
           step_id: 'step_id',
           collect: fieldValue,
           ended: false,
+          type: 'flow',
         }),
       ).toEqual(`not ${fieldValue}`)
     })

rasa/core/channels/inspector/src/helpers/formatters.ts

@@ -1,5 +1,5 @@
-import { rasaColors } from '../theme/base/colors'
-import type { Event, Flow, Slot, Stack } from '../types'
+import {rasaColors} from '../theme/base/colors'
+import type {Event, Flow, Slot, Stack} from '../types'
 
 export function formatSlots(slots: { [key: string]: unknown }) {
   if (!slots) {
@@ -13,7 +13,7 @@ export function formatSlots(slots: { [key: string]: unknown }) {
         slotDuple[0] !== 'flow_hashes' &&
         slotDuple[1] != null,
     )
-    .map((slotDuple) => ({ name: slotDuple[0], value: slotDuple[1] }))
+    .map((slotDuple) => ({name: slotDuple[0], value: slotDuple[1]}))
 }
 
 export const formatTestCases = (events: Event[], sessionId: string) => {
@@ -85,11 +85,14 @@ export const formatFlow = (
     `flowchart TD
 classDef collect stroke-width:1px
 classDef action fill:#FBFCFD,stroke:#A0B8CF
+classDef noop fill:#FBFCFD
+classDef callstep fill:#FBFCFD
 classDef link fill:#f43
 classDef slot fill:#e8f3db,stroke:#c5e1a5
 classDef endstep fill:#ccc,stroke:#444
 classDef previous stroke:${rasaColors.rasaOrange[400]},stroke-width:1px
 classDef active stroke:${rasaColors.rasaOrange[400]},stroke-width:3px,fill:${rasaColors.warning[50]}
+classDef pulse animation:pulse 2s infinite
 `,
   ]
   try {
@@ -210,6 +213,24 @@ function renderStepSequence(
       )}"]:::slot\n`
     }
 
+    if (step.call) {
+      const isAgentWaitingOnInput = !!currentStack?.agent_id && currentStack?.state === 'waiting_for_input'
+      mermaidTextFragment += `${mermaidId}["${encodeDoubleQuotes(
+        stepId,
+      )} 🤖"]:::callstep\n`
+
+      if (isAgentWaitingOnInput) {
+        mermaidTextFragment += `class ${mermaidId} pulse\n`
+      }
+    }
+
+    if (step.noop) {
+      mermaidTextFragment += `${mermaidId}["${parseFieldUsingStack(
+        stepId,
+        currentStack
+      )}"]:::noop\n`
+    }
+
     if (activeStep && stepId === activeStep) {
       mermaidTextFragment += `class ${mermaidId} active\n`
     } else if (stepTrail?.includes(stepId)) {

rasa/core/channels/inspector/src/helpers/utils.test.ts

@@ -0,0 +1,127 @@
+import { updateAgentsStatus } from './utils'
+import type { Agent, Event } from '../types'
+
+// Helper to clone agents (immutability check)
+const clone = <T>(x: T): T => JSON.parse(JSON.stringify(x))
+
+describe('updateAgentsStatus', () => {
+  test('returns empty array when agents undefined', () => {
+    // @ts-expect-error testing runtime behavior with undefined
+    expect(updateAgentsStatus(undefined, [])).toEqual([])
+  })
+
+  test('returns same agents when agents empty', () => {
+    const agents: Agent[] = []
+    const events: Event[] = [
+      { event: 'agent_started', timestamp: '1' } as any,
+    ]
+    expect(updateAgentsStatus(agents, events)).toEqual([])
+  })
+
+  test('returns same agents when events empty', () => {
+    const agents: Agent[] = [
+      { name: 'alpha', status: 'completed' },
+    ]
+    const events: Event[] = []
+    expect(updateAgentsStatus(clone(agents), events)).toEqual(agents)
+  })
+
+  test('ignores non-agent events', () => {
+    const agents: Agent[] = [
+      { name: 'alpha', status: 'completed' },
+    ]
+    const events: Event[] = [
+      { event: 'user', text: 'hi', timestamp: '1' } as any,
+      { event: 'bot', text: 'hey', timestamp: '2' } as any,
+    ]
+    expect(updateAgentsStatus(clone(agents), events)).toEqual(agents)
+  })
+
+  test('uses latest event by timestamp per agent', () => {
+    const agents: Agent[] = [
+      { name: 'alpha', status: 'completed' },
+      { name: 'beta', status: 'completed' },
+    ]
+
+    const events: Event[] = [
+      { event: 'agent_started', agent_id: 'alpha', timestamp: 1 } as any,
+      { event: 'agent_interrupted', agent_id: 'alpha', timestamp: 2 } as any,
+      { event: 'agent_resumed', agent_id: 'alpha', timestamp: 3 } as any,
+      { event: 'agent_cancelled', agent_id: 'beta', timestamp: 5 } as any,
+    ]
+
+    const result = updateAgentsStatus(clone(agents), events)
+    expect(result).toEqual([
+      { name: 'alpha', status: 'running' },
+      { name: 'beta', status: 'cancelled' },
+    ])
+  })
+
+  test('later event wins when timestamps missing', () => {
+    const agents: Agent[] = [
+      { name: 'alpha', status: 'completed' },
+    ]
+
+    const events: Event[] = [
+      { event: 'agent_started', agent_id: 'alpha', timestamp: null } as any,
+      { event: 'agent_interrupted', agent_id: 'alpha', timestamp: null } as any,
+    ]
+
+    const result = updateAgentsStatus(clone(agents), events)
+    expect(result).toEqual([
+      { name: 'alpha', status: 'interrupted' },
+    ])
+  })
+
+  test('agent_completed maps to provided status and falls back to completed', () => {
+    const agents: Agent[] = [
+      { name: 'alpha', status: 'running' },
+      { name: 'beta', status: 'running' },
+    ]
+
+    const events: Event[] = [
+      { event: 'agent_completed', agent_id: 'alpha', status: 'completed', timestamp: 10 } as any,
+      { event: 'agent_completed', agent_id: 'beta', timestamp: 10 } as any,
+    ]
+
+    const result = updateAgentsStatus(clone(agents), events)
+    expect(result).toEqual([
+      { name: 'alpha', status: 'completed' },
+      { name: 'beta', status: 'completed' },
+    ])
+  })
+
+  test('unmatched agents remain unchanged', () => {
+    const agents: Agent[] = [
+      { name: 'alpha', status: 'running' },
+      { name: 'gamma', status: 'interrupted' },
+    ]
+
+    const events: Event[] = [
+      { event: 'agent_resumed', agent_id: 'alpha', timestamp: 1 } as any,
+      { event: 'agent_cancelled', agent_id: 'beta', timestamp: 2 } as any, // beta not in agents list
+    ]
+
+    const result = updateAgentsStatus(clone(agents), events)
+    expect(result).toEqual([
+      { name: 'alpha', status: 'running' },
+      { name: 'gamma', status: 'interrupted' },
+    ])
+  })
+
+  test('handles mixed timestamp types (string/number)', () => {
+    const agents: Agent[] = [
+      { name: 'alpha', status: 'completed' },
+    ]
+
+    const events: Event[] = [
+      { event: 'agent_started', agent_id: 'alpha', timestamp: '1' } as any,
+      { event: 'agent_interrupted', agent_id: 'alpha', timestamp: 2 } as any,
+    ]
+
+    const result = updateAgentsStatus(clone(agents), events)
+    expect(result).toEqual([
+      { name: 'alpha', status: 'interrupted' },
+    ])
+  })
+})

rasa/core/channels/inspector/src/helpers/utils.ts

@@ -1,4 +1,4 @@
-import { SelectedStack, Stack, Event } from '../types'
+import {SelectedStack, Stack, Event, Agent, AGENT_EVENT_TYPES} from '../types'
 import { immutableJSONPatch } from 'immutable-json-patch'
 
 export const shouldShowTooltip = (text: string) => {
@@ -146,3 +146,68 @@ export const updatedActiveFrame = (
     return previous
   }
 }
+
+export function updateAgentsStatus(agents: Agent[], events: Event[]): Agent[] {
+  if (!agents || agents.length === 0 || !events || events.length === 0) {
+    return agents || []
+  }
+
+  const agentEventTypes = new Set<Event['event']>(AGENT_EVENT_TYPES as unknown as Event['event'][] )
+
+  type AgentEventLite = Agent & {
+    agent_id: string
+    event: Event['event']
+    timestamp?: string | number | null
+  }
+  
+  const latestByAgent = new Map<string, AgentEventLite>()
+
+  for (let i = 0; i < events.length; i++) {
+    const event = events[i] as unknown as AgentEventLite
+    if (!event || !agentEventTypes.has(event.event)) continue
+
+    const agentId = event.agent_id
+    if (!agentId) continue
+
+    const previousEvent = latestByAgent.get(agentId)
+
+    const currentTimestamp = event.timestamp == null ? undefined : Number(event.timestamp)
+    const previousTimestamp = previousEvent?.timestamp == null ? undefined : Number(previousEvent.timestamp)
+
+    const isNewer =
+      previousEvent == null ||
+      (currentTimestamp != null && previousTimestamp != null && currentTimestamp > previousTimestamp) ||
+      (currentTimestamp != null && previousTimestamp == null)
+    // If both timestamps are missing, prefer later in the array (current)
+
+    if (isNewer || (currentTimestamp == null && previousTimestamp == null)) {
+      latestByAgent.set(agentId, event)
+    }
+  }
+
+  return agents.map((agent) => {
+    const latest = latestByAgent.get(agent.name)
+    if (!latest) return agent
+
+    let status = agent.status
+    switch (latest.event) {
+      case 'agent_started':
+      case 'agent_resumed':
+        status = 'running'
+        break
+      case 'agent_interrupted':
+        status = 'interrupted'
+        break
+      case 'agent_cancelled':
+        status = 'cancelled'
+        break
+      case 'agent_completed':
+        status = latest.status || 'completed'
+        break
+      default:
+        break
+    }
+
+    return { ...agent, status }
+  })
+}