stream-chat-react-native-core 9.10.2-beta.4 → 9.11.0-beta.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (183) hide show
  1. package/lib/commonjs/ai-components/README.md +50 -2
  2. package/lib/commonjs/ai-components/index.js +24 -0
  3. package/lib/commonjs/ai-components/index.js.map +1 -1
  4. package/lib/commonjs/ai-components/message-composer/ComposerView.js +4 -2
  5. package/lib/commonjs/ai-components/message-composer/ComposerView.js.map +1 -1
  6. package/lib/commonjs/ai-components/message-composer/SpeechToTextButton.js +4 -1
  7. package/lib/commonjs/ai-components/message-composer/SpeechToTextButton.js.map +1 -1
  8. package/lib/commonjs/ai-components/native/AIDictation.js +6 -0
  9. package/lib/commonjs/ai-components/native/AIDictation.js.map +1 -1
  10. package/lib/commonjs/ai-components/store/dictation/store.js +4 -2
  11. package/lib/commonjs/ai-components/store/dictation/store.js.map +1 -1
  12. package/lib/commonjs/ai-components/store/dictation/useDictationState.js +4 -2
  13. package/lib/commonjs/ai-components/store/dictation/useDictationState.js.map +1 -1
  14. package/lib/commonjs/ai-components/store/index.js +22 -0
  15. package/lib/commonjs/ai-components/store/index.js.map +1 -1
  16. package/lib/commonjs/ai-components/theme.js +3 -0
  17. package/lib/commonjs/ai-components/theme.js.map +1 -1
  18. package/lib/commonjs/ai-components/transcription/useDictation.js +7 -3
  19. package/lib/commonjs/ai-components/transcription/useDictation.js.map +1 -1
  20. package/lib/commonjs/ai-components/transcription/useDictationAudioLevel.js +32 -0
  21. package/lib/commonjs/ai-components/transcription/useDictationAudioLevel.js.map +1 -0
  22. package/lib/commonjs/ai-components/voice-visualization/VoiceGlow.js +79 -0
  23. package/lib/commonjs/ai-components/voice-visualization/VoiceGlow.js.map +1 -0
  24. package/lib/commonjs/ai-components/voice-visualization/VoiceVisualizationView.js +147 -0
  25. package/lib/commonjs/ai-components/voice-visualization/VoiceVisualizationView.js.map +1 -0
  26. package/lib/commonjs/ai-components/voice-visualization/VoiceWave.js +58 -0
  27. package/lib/commonjs/ai-components/voice-visualization/VoiceWave.js.map +1 -0
  28. package/lib/commonjs/ai-components/voice-visualization/bounce.js +60 -0
  29. package/lib/commonjs/ai-components/voice-visualization/bounce.js.map +1 -0
  30. package/lib/commonjs/ai-components/voice-visualization/driver.js +208 -0
  31. package/lib/commonjs/ai-components/voice-visualization/driver.js.map +1 -0
  32. package/lib/commonjs/ai-components/voice-visualization/glowShader.js +218 -0
  33. package/lib/commonjs/ai-components/voice-visualization/glowShader.js.map +1 -0
  34. package/lib/commonjs/ai-components/voice-visualization/index.js +18 -0
  35. package/lib/commonjs/ai-components/voice-visualization/index.js.map +1 -0
  36. package/lib/commonjs/ai-components/voice-visualization/math.js +70 -0
  37. package/lib/commonjs/ai-components/voice-visualization/math.js.map +1 -0
  38. package/lib/commonjs/ai-components/voice-visualization/palette.js +47 -0
  39. package/lib/commonjs/ai-components/voice-visualization/palette.js.map +1 -0
  40. package/lib/commonjs/ai-components/voice-visualization/simplex.js +77 -0
  41. package/lib/commonjs/ai-components/voice-visualization/simplex.js.map +1 -0
  42. package/lib/commonjs/ai-components/voice-visualization/styles.js +33 -0
  43. package/lib/commonjs/ai-components/voice-visualization/styles.js.map +1 -0
  44. package/lib/commonjs/ai-components/voice-visualization/types.js +4 -0
  45. package/lib/commonjs/ai-components/voice-visualization/types.js.map +1 -0
  46. package/lib/commonjs/ai-components/voice-visualization/useReduceMotion.js +28 -0
  47. package/lib/commonjs/ai-components/voice-visualization/useReduceMotion.js.map +1 -0
  48. package/lib/commonjs/ai-components/voice-visualization/useVoiceVisualization.js +129 -0
  49. package/lib/commonjs/ai-components/voice-visualization/useVoiceVisualization.js.map +1 -0
  50. package/lib/commonjs/ai-components/voice-visualization/waveGeometry.js +124 -0
  51. package/lib/commonjs/ai-components/voice-visualization/waveGeometry.js.map +1 -0
  52. package/lib/commonjs/version.json +1 -1
  53. package/lib/module/ai-components/README.md +50 -2
  54. package/lib/module/ai-components/index.js +24 -0
  55. package/lib/module/ai-components/index.js.map +1 -1
  56. package/lib/module/ai-components/message-composer/ComposerView.js +4 -2
  57. package/lib/module/ai-components/message-composer/ComposerView.js.map +1 -1
  58. package/lib/module/ai-components/message-composer/SpeechToTextButton.js +4 -1
  59. package/lib/module/ai-components/message-composer/SpeechToTextButton.js.map +1 -1
  60. package/lib/module/ai-components/native/AIDictation.js +6 -0
  61. package/lib/module/ai-components/native/AIDictation.js.map +1 -1
  62. package/lib/module/ai-components/store/dictation/store.js +4 -2
  63. package/lib/module/ai-components/store/dictation/store.js.map +1 -1
  64. package/lib/module/ai-components/store/dictation/useDictationState.js +4 -2
  65. package/lib/module/ai-components/store/dictation/useDictationState.js.map +1 -1
  66. package/lib/module/ai-components/store/index.js +22 -0
  67. package/lib/module/ai-components/store/index.js.map +1 -1
  68. package/lib/module/ai-components/theme.js +3 -0
  69. package/lib/module/ai-components/theme.js.map +1 -1
  70. package/lib/module/ai-components/transcription/useDictation.js +7 -3
  71. package/lib/module/ai-components/transcription/useDictation.js.map +1 -1
  72. package/lib/module/ai-components/transcription/useDictationAudioLevel.js +32 -0
  73. package/lib/module/ai-components/transcription/useDictationAudioLevel.js.map +1 -0
  74. package/lib/module/ai-components/voice-visualization/VoiceGlow.js +79 -0
  75. package/lib/module/ai-components/voice-visualization/VoiceGlow.js.map +1 -0
  76. package/lib/module/ai-components/voice-visualization/VoiceVisualizationView.js +147 -0
  77. package/lib/module/ai-components/voice-visualization/VoiceVisualizationView.js.map +1 -0
  78. package/lib/module/ai-components/voice-visualization/VoiceWave.js +58 -0
  79. package/lib/module/ai-components/voice-visualization/VoiceWave.js.map +1 -0
  80. package/lib/module/ai-components/voice-visualization/bounce.js +60 -0
  81. package/lib/module/ai-components/voice-visualization/bounce.js.map +1 -0
  82. package/lib/module/ai-components/voice-visualization/driver.js +208 -0
  83. package/lib/module/ai-components/voice-visualization/driver.js.map +1 -0
  84. package/lib/module/ai-components/voice-visualization/glowShader.js +218 -0
  85. package/lib/module/ai-components/voice-visualization/glowShader.js.map +1 -0
  86. package/lib/module/ai-components/voice-visualization/index.js +18 -0
  87. package/lib/module/ai-components/voice-visualization/index.js.map +1 -0
  88. package/lib/module/ai-components/voice-visualization/math.js +70 -0
  89. package/lib/module/ai-components/voice-visualization/math.js.map +1 -0
  90. package/lib/module/ai-components/voice-visualization/palette.js +47 -0
  91. package/lib/module/ai-components/voice-visualization/palette.js.map +1 -0
  92. package/lib/module/ai-components/voice-visualization/simplex.js +77 -0
  93. package/lib/module/ai-components/voice-visualization/simplex.js.map +1 -0
  94. package/lib/module/ai-components/voice-visualization/styles.js +33 -0
  95. package/lib/module/ai-components/voice-visualization/styles.js.map +1 -0
  96. package/lib/module/ai-components/voice-visualization/types.js +4 -0
  97. package/lib/module/ai-components/voice-visualization/types.js.map +1 -0
  98. package/lib/module/ai-components/voice-visualization/useReduceMotion.js +28 -0
  99. package/lib/module/ai-components/voice-visualization/useReduceMotion.js.map +1 -0
  100. package/lib/module/ai-components/voice-visualization/useVoiceVisualization.js +129 -0
  101. package/lib/module/ai-components/voice-visualization/useVoiceVisualization.js.map +1 -0
  102. package/lib/module/ai-components/voice-visualization/waveGeometry.js +124 -0
  103. package/lib/module/ai-components/voice-visualization/waveGeometry.js.map +1 -0
  104. package/lib/module/version.json +1 -1
  105. package/lib/typescript/ai-components/index.d.ts +3 -1
  106. package/lib/typescript/ai-components/index.d.ts.map +1 -1
  107. package/lib/typescript/ai-components/message-composer/ComposerView.d.ts.map +1 -1
  108. package/lib/typescript/ai-components/native/AIDictation.d.ts +6 -0
  109. package/lib/typescript/ai-components/native/AIDictation.d.ts.map +1 -1
  110. package/lib/typescript/ai-components/store/dictation/store.d.ts +7 -0
  111. package/lib/typescript/ai-components/store/dictation/store.d.ts.map +1 -1
  112. package/lib/typescript/ai-components/store/dictation/useDictationState.d.ts +1 -0
  113. package/lib/typescript/ai-components/store/dictation/useDictationState.d.ts.map +1 -1
  114. package/lib/typescript/ai-components/store/index.d.ts +2 -0
  115. package/lib/typescript/ai-components/store/index.d.ts.map +1 -1
  116. package/lib/typescript/ai-components/theme.d.ts +13 -0
  117. package/lib/typescript/ai-components/theme.d.ts.map +1 -1
  118. package/lib/typescript/ai-components/transcription/useDictation.d.ts +9 -1
  119. package/lib/typescript/ai-components/transcription/useDictation.d.ts.map +1 -1
  120. package/lib/typescript/ai-components/transcription/useDictationAudioLevel.d.ts +11 -0
  121. package/lib/typescript/ai-components/transcription/useDictationAudioLevel.d.ts.map +1 -0
  122. package/lib/typescript/ai-components/voice-visualization/VoiceGlow.d.ts +13 -0
  123. package/lib/typescript/ai-components/voice-visualization/VoiceGlow.d.ts.map +1 -0
  124. package/lib/typescript/ai-components/voice-visualization/VoiceVisualizationView.d.ts +26 -0
  125. package/lib/typescript/ai-components/voice-visualization/VoiceVisualizationView.d.ts.map +1 -0
  126. package/lib/typescript/ai-components/voice-visualization/VoiceWave.d.ts +13 -0
  127. package/lib/typescript/ai-components/voice-visualization/VoiceWave.d.ts.map +1 -0
  128. package/lib/typescript/ai-components/voice-visualization/bounce.d.ts +20 -0
  129. package/lib/typescript/ai-components/voice-visualization/bounce.d.ts.map +1 -0
  130. package/lib/typescript/ai-components/voice-visualization/driver.d.ts +96 -0
  131. package/lib/typescript/ai-components/voice-visualization/driver.d.ts.map +1 -0
  132. package/lib/typescript/ai-components/voice-visualization/glowShader.d.ts +7 -0
  133. package/lib/typescript/ai-components/voice-visualization/glowShader.d.ts.map +1 -0
  134. package/lib/typescript/ai-components/voice-visualization/index.d.ts +4 -0
  135. package/lib/typescript/ai-components/voice-visualization/index.d.ts.map +1 -0
  136. package/lib/typescript/ai-components/voice-visualization/math.d.ts +26 -0
  137. package/lib/typescript/ai-components/voice-visualization/math.d.ts.map +1 -0
  138. package/lib/typescript/ai-components/voice-visualization/palette.d.ts +18 -0
  139. package/lib/typescript/ai-components/voice-visualization/palette.d.ts.map +1 -0
  140. package/lib/typescript/ai-components/voice-visualization/simplex.d.ts +11 -0
  141. package/lib/typescript/ai-components/voice-visualization/simplex.d.ts.map +1 -0
  142. package/lib/typescript/ai-components/voice-visualization/styles.d.ts +24 -0
  143. package/lib/typescript/ai-components/voice-visualization/styles.d.ts.map +1 -0
  144. package/lib/typescript/ai-components/voice-visualization/types.d.ts +45 -0
  145. package/lib/typescript/ai-components/voice-visualization/types.d.ts.map +1 -0
  146. package/lib/typescript/ai-components/voice-visualization/useReduceMotion.d.ts +10 -0
  147. package/lib/typescript/ai-components/voice-visualization/useReduceMotion.d.ts.map +1 -0
  148. package/lib/typescript/ai-components/voice-visualization/useVoiceVisualization.d.ts +18 -0
  149. package/lib/typescript/ai-components/voice-visualization/useVoiceVisualization.d.ts.map +1 -0
  150. package/lib/typescript/ai-components/voice-visualization/waveGeometry.d.ts +25 -0
  151. package/lib/typescript/ai-components/voice-visualization/waveGeometry.d.ts.map +1 -0
  152. package/package.json +1 -1
  153. package/src/ai-components/README.md +50 -2
  154. package/src/ai-components/__tests__/VoiceVisualization.test.tsx +208 -0
  155. package/src/ai-components/__tests__/dictation.test.tsx +58 -1
  156. package/src/ai-components/__tests__/voiceDriver.test.ts +317 -0
  157. package/src/ai-components/__tests__/voicePalette.test.ts +124 -0
  158. package/src/ai-components/index.ts +3 -1
  159. package/src/ai-components/message-composer/ComposerView.tsx +6 -3
  160. package/src/ai-components/message-composer/SpeechToTextButton.tsx +5 -2
  161. package/src/ai-components/native/AIDictation.ts +11 -0
  162. package/src/ai-components/store/dictation/store.ts +9 -0
  163. package/src/ai-components/store/dictation/useDictationState.ts +2 -1
  164. package/src/ai-components/store/index.ts +2 -0
  165. package/src/ai-components/theme.ts +16 -0
  166. package/src/ai-components/transcription/useDictation.ts +16 -3
  167. package/src/ai-components/transcription/useDictationAudioLevel.ts +38 -0
  168. package/src/ai-components/voice-visualization/VoiceGlow.tsx +79 -0
  169. package/src/ai-components/voice-visualization/VoiceVisualizationView.tsx +179 -0
  170. package/src/ai-components/voice-visualization/VoiceWave.tsx +67 -0
  171. package/src/ai-components/voice-visualization/bounce.ts +61 -0
  172. package/src/ai-components/voice-visualization/driver.ts +240 -0
  173. package/src/ai-components/voice-visualization/glowShader.ts +227 -0
  174. package/src/ai-components/voice-visualization/index.ts +3 -0
  175. package/src/ai-components/voice-visualization/math.ts +73 -0
  176. package/src/ai-components/voice-visualization/palette.ts +59 -0
  177. package/src/ai-components/voice-visualization/simplex.ts +92 -0
  178. package/src/ai-components/voice-visualization/styles.ts +42 -0
  179. package/src/ai-components/voice-visualization/types.ts +44 -0
  180. package/src/ai-components/voice-visualization/useReduceMotion.ts +34 -0
  181. package/src/ai-components/voice-visualization/useVoiceVisualization.ts +153 -0
  182. package/src/ai-components/voice-visualization/waveGeometry.ts +137 -0
  183. package/src/version.json +1 -1
@@ -0,0 +1,124 @@
1
+ import { audioLevelInput, expressivityInput, speechEnv } from '../voice-visualization/math';
2
+ import { luminance, parseColor, rgbToHsv, toHsvPalette } from '../voice-visualization/palette';
3
+ import { snoise } from '../voice-visualization/simplex';
4
+ import { GLOW_STYLE, WAVE_STYLE } from '../voice-visualization/styles';
5
+
6
+ describe('voice visualization palette', () => {
7
+ it('keeps a color’s full hue, saturation and value', () => {
8
+ const hsv = rgbToHsv([0, 95 / 255, 1]);
9
+
10
+ expect(hsv.hue * 360).toBeCloseTo(217.6, 1);
11
+ expect(hsv.saturation).toBeCloseTo(1, 4);
12
+ expect(hsv.value).toBeCloseTo(1, 4);
13
+ });
14
+
15
+ it('keeps a muted color muted', () => {
16
+ // hsv(108°, 25%, 60%)
17
+ const hsv = rgbToHsv([0.48, 0.6, 0.45]);
18
+
19
+ expect(hsv.hue).toBeCloseTo(0.3, 2);
20
+ expect(hsv.saturation).toBeCloseTo(0.25, 2);
21
+ expect(hsv.value).toBeCloseTo(0.6, 4);
22
+ });
23
+
24
+ it('gives a gray no saturation', () => {
25
+ const hsv = rgbToHsv([0.5, 0.5, 0.5]);
26
+
27
+ expect(hsv.saturation).toBe(0);
28
+ expect(hsv.value).toBe(0.5);
29
+ });
30
+
31
+ it.each([
32
+ ['#005fff', [0, 95 / 255, 1]],
33
+ ['#0af', [0, 170 / 255, 1]],
34
+ ['#005fff80', [0, 95 / 255, 1]],
35
+ ['rgb(0, 95, 255)', [0, 95 / 255, 1]],
36
+ ['rgba(0, 95, 255, 0.5)', [0, 95 / 255, 1]],
37
+ ['hsl(0, 100%, 50%)', [1, 0, 0]],
38
+ ['rebeccapurple', [0.4, 0.2, 0.6]],
39
+ ['#000000', [0, 0, 0]],
40
+ ])('reads %s', (color, expected) => {
41
+ const rgb = parseColor(color);
42
+ expect(rgb).not.toBeNull();
43
+ rgb?.forEach((channel, i) => expect(channel).toBeCloseTo(expected[i], 2));
44
+ });
45
+
46
+ it('skips colors it cannot read', () => {
47
+ expect(parseColor('not-a-color')).toBeNull();
48
+ expect(parseColor('#12345')).toBeNull();
49
+ expect(parseColor('')).toBeNull();
50
+ expect(toHsvPalette(['nope', '#ff0000'])).toHaveLength(1);
51
+ });
52
+
53
+ it('keeps only the first five colors', () => {
54
+ expect(toHsvPalette(Array.from({ length: 7 }, () => '#ff0000'))).toHaveLength(5);
55
+ });
56
+
57
+ it('measures luminance for the light or dark look', () => {
58
+ expect(luminance('#ffffff')).toBeCloseTo(1, 4);
59
+ expect(luminance('#000000')).toBe(0);
60
+ expect(luminance('#1c1e22')).toBeLessThan(0.5);
61
+ expect(luminance('')).toBeNull();
62
+ });
63
+ });
64
+
65
+ describe('voice visualization inputs', () => {
66
+ it('clamps expressivity to 0-2 and replaces one that is not a number', () => {
67
+ expect(expressivityInput(3)).toBe(2);
68
+ expect(expressivityInput(-1)).toBe(0);
69
+ expect(expressivityInput(0.5)).toBe(0.5);
70
+ expect(expressivityInput(Number.NaN)).toBe(1);
71
+ expect(expressivityInput(Number.POSITIVE_INFINITY)).toBe(1);
72
+ expect(expressivityInput(undefined)).toBe(1);
73
+ });
74
+
75
+ it('clamps the audio level to 0-1 and treats one that is not a number as no audio', () => {
76
+ expect(audioLevelInput(2)).toBe(1);
77
+ expect(audioLevelInput(-1)).toBe(0);
78
+ expect(audioLevelInput(0.4)).toBe(0.4);
79
+ expect(audioLevelInput(Number.NaN)).toBeNull();
80
+ expect(audioLevelInput(null)).toBeNull();
81
+ expect(audioLevelInput(undefined)).toBeNull();
82
+ });
83
+ });
84
+
85
+ describe('voice visualization noise', () => {
86
+ it('matches the shader’s simplex noise as a GPU computes it', () => {
87
+ // Read back from the shader's own snoise on a GPU (stream-chat-react's fixtures).
88
+ const gpu: Array<[number, number, number, number]> = [
89
+ [0.3, 21, 0, 0.7696],
90
+ [1.7, 5, 0, 0.208],
91
+ [0.12, 0, 0, 0.049],
92
+ [2.5, 9, 0, 0.3869],
93
+ [10.2, 14, 0, 0.603],
94
+ [3.3, 1.1, 0.9, -0.7541],
95
+ [0.05, 0.05, 0.05, -0.5481],
96
+ ];
97
+ for (const [x, y, z, expected] of gpu) {
98
+ expect(snoise(x, y, z)).toBeCloseTo(expected, 2);
99
+ }
100
+ });
101
+
102
+ it('gives speech phrases and pauses', () => {
103
+ const samples = Array.from({ length: 2000 }, (_, i) => speechEnv(i * 0.01));
104
+
105
+ expect(Math.min(...samples)).toBe(0);
106
+ expect(Math.max(...samples)).toBeGreaterThan(0.6);
107
+ expect(Math.max(...samples)).toBeLessThanOrEqual(1);
108
+ });
109
+ });
110
+
111
+ describe('voice visualization hit areas', () => {
112
+ it('takes the globe within a third of the shorter side', () => {
113
+ expect(GLOW_STYLE.contains(0, 0)).toBe(true);
114
+ expect(GLOW_STYLE.contains(0.2, 0.2)).toBe(true);
115
+ expect(GLOW_STYLE.contains(0.3, 0.3)).toBe(false);
116
+ });
117
+
118
+ it('takes the string across, and only near its line', () => {
119
+ expect(WAVE_STYLE.contains(0.45, 0)).toBe(true);
120
+ expect(WAVE_STYLE.contains(0, 0.1)).toBe(true);
121
+ expect(WAVE_STYLE.contains(0, 0.2)).toBe(false);
122
+ expect(WAVE_STYLE.contains(0.55, 0)).toBe(false);
123
+ });
124
+ });
@@ -5,6 +5,7 @@ export * from './charts';
5
5
  export * from './message-composer';
6
6
  export * from './StreamingMessageView';
7
7
  export * from './AITypingIndicatorView';
8
+ export * from './voice-visualization';
8
9
 
9
10
  // components
10
11
  export * from './components';
@@ -15,7 +16,8 @@ export * from './contexts';
15
16
  // services
16
17
  export * from './services';
17
18
  export * from './transcription/useDictation';
18
- export { isDictationAvailable } from './native/AIDictation';
19
+ export * from './transcription/useDictationAudioLevel';
20
+ export { isDictationAvailable, type DictationLevel } from './native/AIDictation';
19
21
 
20
22
  // state-stores
21
23
  export * from './store';
@@ -19,7 +19,7 @@ import type {
19
19
  AbstractMediaPickerService,
20
20
  MediaPickerState,
21
21
  } from '../services/media-picker-service/AbstractMediaPickerService';
22
- import { useComposerText } from '../store';
22
+ import { COMPOSER_DICTATION_SOURCE, useComposerText } from '../store';
23
23
  import type { ComposerState } from '../store';
24
24
  import { useDictationState } from '../store/dictation/useDictationState';
25
25
  import { useDictationTranscript } from '../store/dictation/useDictationTranscript';
@@ -100,7 +100,7 @@ export const AIMessageInput = () => {
100
100
  } = useTheme();
101
101
  const styles = useStyles();
102
102
 
103
- const { error, isRecording } = useDictationState();
103
+ const { error, isRecording, source } = useDictationState();
104
104
  const { text } = useComposerText();
105
105
  const { transcript } = useDictationTranscript();
106
106
  const { setText } = useMessageComposerContext();
@@ -109,10 +109,13 @@ export const AIMessageInput = () => {
109
109
  useEffect(() => {
110
110
  const transcriptChanged = transcript !== previousTranscript.current;
111
111
  previousTranscript.current = transcript;
112
+ // Only dictation from the composer's own button fills the input; dictation started elsewhere
113
+ // (a voice mode, say) must not overwrite what the user typed.
114
+ if (source !== COMPOSER_DICTATION_SOURCE) return;
112
115
  if (!error && (isRecording || transcriptChanged)) {
113
116
  setText(transcript);
114
117
  }
115
- }, [isRecording, transcript, error, setText]);
118
+ }, [isRecording, transcript, error, setText, source]);
116
119
 
117
120
  return (
118
121
  <TextInput
@@ -9,7 +9,7 @@ import { Mic } from '../internal/icons/Mic';
9
9
 
10
10
  import { isDictationAvailable, warnIfDictationUnavailable } from '../native/AIDictation';
11
11
  import type { DictationStartOptions } from '../native-specs/NativeAIDictation';
12
- import { dictationStore, type DictationStoreState } from '../store';
12
+ import { COMPOSER_DICTATION_SOURCE, dictationStore, type DictationStoreState } from '../store';
13
13
  import { useDictation } from '../transcription/useDictation';
14
14
 
15
15
  const selector = ({ isRecording }: DictationStoreState) => ({
@@ -33,6 +33,9 @@ export const SpeechToTextButton = (props: SpeechToTextButtonProps) => {
33
33
  return <AvailableSpeechToTextButton {...props} />;
34
34
  };
35
35
 
36
+ // Marks this button's dictation as the composer's, so `ComposerView` fills its input with it.
37
+ const DICTATION_OPTIONS = { source: COMPOSER_DICTATION_SOURCE };
38
+
36
39
  const AvailableSpeechToTextButton = ({ options }: SpeechToTextButtonProps) => {
37
40
  const {
38
41
  theme: {
@@ -41,7 +44,7 @@ const AvailableSpeechToTextButton = ({ options }: SpeechToTextButtonProps) => {
41
44
  },
42
45
  } = useTheme();
43
46
 
44
- const { cancel, toggle } = useDictation(options);
47
+ const { cancel, toggle } = useDictation(options, DICTATION_OPTIONS);
45
48
  const { isRecording } = useStateStore(dictationStore, selector);
46
49
 
47
50
  useEffect(() => {
@@ -11,6 +11,10 @@ import NativeAIDictation, {
11
11
  const ON_STATE_EVENT = 'AIDictationOnState';
12
12
  const ON_RESULT_EVENT = 'AIDictationOnResult';
13
13
  const ON_ERROR_EVENT = 'AIDictationOnError';
14
+ const ON_LEVEL_EVENT = 'AIDictationOnLevel';
15
+
16
+ /** The microphone's level while dictation listens, from 0 to 1. */
17
+ export type DictationLevel = { level: number };
14
18
 
15
19
  let eventEmitter: NativeEventEmitter | undefined;
16
20
  const getEventEmitter = () => {
@@ -74,4 +78,11 @@ export const AIDictation = {
74
78
  );
75
79
  return sub ? () => sub.remove() : noop;
76
80
  },
81
+ /** The microphone's level while dictation listens, about 10 to 30 times a second. */
82
+ addLevelListener(listener: (payload: DictationLevel) => void) {
83
+ const sub = getEventEmitter()?.addListener(ON_LEVEL_EVENT, (event) =>
84
+ listener(event as DictationLevel),
85
+ );
86
+ return sub ? () => sub.remove() : noop;
87
+ },
77
88
  };
@@ -7,6 +7,11 @@ export type DictationStoreState = {
7
7
  transcript: string;
8
8
  isRecording: boolean;
9
9
  error?: DictationError | null;
10
+ /**
11
+ * Who started the current dictation, as passed to `useDictation`, or `null`. Lets a consumer
12
+ * such as `ComposerView` pick up only the transcripts meant for it.
13
+ */
14
+ source?: string | null;
10
15
  };
11
16
 
12
17
  const DEFAULT_STATE: DictationStoreState = {
@@ -14,8 +19,12 @@ const DEFAULT_STATE: DictationStoreState = {
14
19
  transcript: '',
15
20
  isRecording: false,
16
21
  error: null,
22
+ source: null,
17
23
  };
18
24
 
25
+ /** The `source` of dictation started by the composer's own `SpeechToTextButton`. */
26
+ export const COMPOSER_DICTATION_SOURCE = 'composer';
27
+
19
28
  export const dictationStore = new StateStore<DictationStoreState>(DEFAULT_STATE);
20
29
 
21
30
  const reset = () => dictationStore.next(DEFAULT_STATE);
@@ -2,10 +2,11 @@ import { useStateStore } from '@stream-io/state-store/react-bindings';
2
2
 
3
3
  import { dictationStore, type DictationStoreState } from './store';
4
4
 
5
- const selector = ({ state, isRecording, error }: DictationStoreState) => ({
5
+ const selector = ({ state, isRecording, error, source }: DictationStoreState) => ({
6
6
  state,
7
7
  isRecording,
8
8
  error,
9
+ source,
9
10
  });
10
11
 
11
12
  export const useDictationState = () => useStateStore(dictationStore, selector);
@@ -9,3 +9,5 @@ export * from './composer/useComposerText';
9
9
 
10
10
  // dictation
11
11
  export * from './dictation/store';
12
+ export * from './dictation/useDictationState';
13
+ export * from './dictation/useDictationTranscript';
@@ -56,6 +56,19 @@ export type Theme = {
56
56
  aiTypingIndicatorView: {
57
57
  text: TextStyle;
58
58
  };
59
+ /**
60
+ * `VoiceGlow` and `VoiceWave`. Optional, so a hand-built `Theme` without it keeps type-checking;
61
+ * the visualizations fall back to the default accent then.
62
+ */
63
+ voiceVisualization?: {
64
+ /** One to five colors, used when a visualization is given no `colors` of its own. */
65
+ colors: string[];
66
+ /**
67
+ * The color the visualizations sit on, which tells them to draw for a light or a dark
68
+ * background. Defaults to the system's color scheme.
69
+ */
70
+ backdropColor?: string;
71
+ };
59
72
  };
60
73
 
61
74
  export const defaultTheme: Theme = {
@@ -135,4 +148,7 @@ export const defaultTheme: Theme = {
135
148
  aiTypingIndicatorView: {
136
149
  text: {},
137
150
  },
151
+ voiceVisualization: {
152
+ colors: [colors.accent_blue],
153
+ },
138
154
  };
@@ -27,7 +27,19 @@ const selector = ({ isRecording }: DictationStoreState) => ({
27
27
  });
28
28
 
29
29
  // TODO: add proper jsdocs
30
- export const useDictation = (defaultOptions: DictationStartOptions = DEFAULT_OPTIONS) => {
30
+ export type UseDictationOptions = {
31
+ /**
32
+ * Who is dictating, stored as the dictation store's `source` when dictation starts. The
33
+ * `ComposerView` only fills its input with dictation whose source is the composer's own, so
34
+ * dictation started elsewhere (a voice mode, say) leaves the composer's text alone.
35
+ */
36
+ source?: string;
37
+ };
38
+
39
+ export const useDictation = (
40
+ defaultOptions: DictationStartOptions = DEFAULT_OPTIONS,
41
+ { source }: UseDictationOptions = {},
42
+ ) => {
31
43
  const { isRecording } = useStateStore(dictationStore, selector);
32
44
 
33
45
  useEffect(() => {
@@ -65,11 +77,12 @@ export const useDictation = (defaultOptions: DictationStartOptions = DEFAULT_OPT
65
77
  code: 'NO_PERMISSION',
66
78
  message: 'Microphone permission not granted',
67
79
  },
80
+ source: source ?? null,
68
81
  });
69
82
  return;
70
83
  }
71
84
 
72
- dictationStore.partialNext({ error: null, transcript: '' });
85
+ dictationStore.partialNext({ error: null, source: source ?? null, transcript: '' });
73
86
 
74
87
  const opts: DictationStartOptions = {
75
88
  ...defaultOptions,
@@ -78,7 +91,7 @@ export const useDictation = (defaultOptions: DictationStartOptions = DEFAULT_OPT
78
91
 
79
92
  AIDictation.start(opts);
80
93
  },
81
- [defaultOptions],
94
+ [defaultOptions, source],
82
95
  );
83
96
 
84
97
  const stop = useCallback(() => {
@@ -0,0 +1,38 @@
1
+ import { useEffect } from 'react';
2
+
3
+ import { type SharedValue, useSharedValue } from 'react-native-reanimated';
4
+
5
+ import { AIDictation } from '../native/AIDictation';
6
+
7
+ /**
8
+ * The microphone's live level while dictation listens, from 0 to 1, for the `audioLevel` of
9
+ * `VoiceWave` and `VoiceGlow`. It is `null` while dictation is not listening, which lets a
10
+ * visualization animate speech on its own.
11
+ *
12
+ * The level is a shared value rather than state, so a level changing many times a second
13
+ * re-renders nothing: the visualizations read it once a frame on the UI thread.
14
+ */
15
+ export const useDictationAudioLevel = (): SharedValue<number | null> => {
16
+ const level = useSharedValue<number | null>(null);
17
+
18
+ useEffect(() => {
19
+ const unsubscribeLevel = AIDictation.addLevelListener((payload) => {
20
+ level.value = typeof payload?.level === 'number' ? payload.level : null;
21
+ });
22
+ const unsubscribeState = AIDictation.addStateListener(({ state }) => {
23
+ if (state !== 'listening' && state !== 'starting') level.value = null;
24
+ });
25
+ const unsubscribeError = AIDictation.addErrorListener(() => {
26
+ level.value = null;
27
+ });
28
+
29
+ return () => {
30
+ unsubscribeLevel();
31
+ unsubscribeState();
32
+ unsubscribeError();
33
+ level.value = null;
34
+ };
35
+ }, [level]);
36
+
37
+ return level;
38
+ };
@@ -0,0 +1,79 @@
1
+ import React from 'react';
2
+
3
+ import { useDerivedValue } from 'react-native-reanimated';
4
+
5
+ import { Canvas, Fill, Shader } from '@shopify/react-native-skia';
6
+
7
+ import { driverReact, driverScale } from './driver';
8
+ import { getGlowEffect } from './glowShader';
9
+ import { GLOW_STYLE } from './styles';
10
+ import type { VoiceVisualizationProps } from './types';
11
+ import type { VoiceVisualization } from './useVoiceVisualization';
12
+ import { type VoiceCanvasSize, VoiceVisualizationView } from './VoiceVisualizationView';
13
+
14
+ export type VoiceGlowProps = VoiceVisualizationProps;
15
+
16
+ const GlowCanvas = ({
17
+ size: { height, width },
18
+ visualization: { dark, driver },
19
+ }: {
20
+ size: VoiceCanvasSize;
21
+ visualization: VoiceVisualization;
22
+ }) => {
23
+ const effect = getGlowEffect();
24
+ const uniforms = useDerivedValue(() => {
25
+ const d = driver.value;
26
+ const color = (i: number) => [d.colors[i].hue, d.colors[i].saturation, d.colors[i].value, 0];
27
+ return {
28
+ bright: d.bright,
29
+ col0: color(0),
30
+ col1: color(1),
31
+ col2: color(2),
32
+ col3: color(3),
33
+ col4: color(4),
34
+ count: d.count,
35
+ dark: dark.value,
36
+ flow: d.flow,
37
+ flowSpin: d.flowSpin,
38
+ hover: [d.hoverX, d.hoverY],
39
+ hoverAmt: d.hoverAmount,
40
+ load: d.load,
41
+ mic: d.mic,
42
+ orbSpin: d.orbSpin,
43
+ react: driverReact(d),
44
+ resolution: [width, height],
45
+ sat: d.sat,
46
+ scale: driverScale(d),
47
+ tap: [d.tapX, d.tapY],
48
+ tapTime: d.tapAge,
49
+ time: d.time,
50
+ voice: d.voice,
51
+ };
52
+ });
53
+
54
+ if (!effect) return null;
55
+
56
+ return (
57
+ <Canvas style={{ height, width }}>
58
+ <Fill>
59
+ <Shader source={effect} uniforms={uniforms} />
60
+ </Fill>
61
+ </Canvas>
62
+ );
63
+ };
64
+
65
+ const renderGlow = (visualization: VoiceVisualization, size: VoiceCanvasSize) => (
66
+ <GlowCanvas size={size} visualization={visualization} />
67
+ );
68
+
69
+ /**
70
+ * A voice agent as a soft, airy globe: color masses orbit inside it and swell with speech, and the
71
+ * gaps between them let the background show through. Tapping the globe bounces it and ripples it
72
+ * from the touch, and so does a hovering mouse or pen.
73
+ *
74
+ * It has no size of its own and draws in the middle of the view it is given, about half as wide
75
+ * as its shorter side, so give it a square.
76
+ */
77
+ export const VoiceGlow = (props: VoiceGlowProps) => (
78
+ <VoiceVisualizationView {...props} renderCanvas={renderGlow} visualStyle={GLOW_STYLE} />
79
+ );
@@ -0,0 +1,179 @@
1
+ import React, { type ReactNode, useCallback, useMemo, useState } from 'react';
2
+ import {
3
+ type GestureResponderEvent,
4
+ type LayoutChangeEvent,
5
+ PixelRatio,
6
+ type PointerEvent,
7
+ StyleSheet,
8
+ View,
9
+ } from 'react-native';
10
+
11
+ import Animated from 'react-native-reanimated';
12
+
13
+ import type { VoiceVisualizationStyle } from './styles';
14
+ import type { VoiceAgentState, VoiceVisualizationProps } from './types';
15
+ import { useVoiceVisualization, type VoiceVisualization } from './useVoiceVisualization';
16
+
17
+ /** What the renderer draws in: the canvas size, in its own units, and the screen's pixels per unit. */
18
+ export type VoiceCanvasSize = { height: number; pixelRatio: number; width: number };
19
+
20
+ type ViewProps = VoiceVisualizationProps & {
21
+ /** Draws the visualization into a canvas of the given size. */
22
+ renderCanvas: (visualization: VoiceVisualization, size: VoiceCanvasSize) => ReactNode;
23
+ visualStyle: VoiceVisualizationStyle;
24
+ };
25
+
26
+ const DEFAULT_STATE_LABELS: Record<VoiceAgentState, string> = {
27
+ connecting: 'Connecting',
28
+ idle: 'Ready',
29
+ listening: 'Listening',
30
+ speaking: 'Speaking',
31
+ thinking: 'Thinking',
32
+ };
33
+
34
+ /**
35
+ * What VoiceWave and VoiceGlow share: the motion, the canvas kept at the view's size and at most
36
+ * the style's density, reduced motion, taps and hover, and accessibility.
37
+ *
38
+ * It names itself "Voice agent" with its state as the value, as on iOS and Android. Touches are
39
+ * only watched, never claimed, so a press handler on the view or a parent still gets them, and
40
+ * only the shape takes them: around it they reach whatever is behind.
41
+ */
42
+ export const VoiceVisualizationView = ({
43
+ accessibilityLabel = 'Voice agent',
44
+ accessibilityValue,
45
+ audioLevel,
46
+ colors,
47
+ expressivity,
48
+ onLayout,
49
+ pointerEvents = 'box-none',
50
+ renderCanvas,
51
+ state,
52
+ stateLabels,
53
+ style,
54
+ visualStyle,
55
+ ...restProps
56
+ }: ViewProps) => {
57
+ const visualization = useVoiceVisualization(
58
+ { audioLevel, colors, expressivity, state },
59
+ visualStyle,
60
+ );
61
+ const { hover, opacity, tap } = visualization;
62
+ const [layout, setLayout] = useState({ height: 0, width: 0 });
63
+
64
+ const handleLayout = useCallback(
65
+ (event: LayoutChangeEvent) => {
66
+ onLayout?.(event);
67
+ const { height, width } = event.nativeEvent.layout;
68
+ setLayout((current) =>
69
+ current.width === width && current.height === height ? current : { height, width },
70
+ );
71
+ },
72
+ [onLayout],
73
+ );
74
+
75
+ const side = Math.min(layout.width, layout.height);
76
+ const pixelRatio = PixelRatio.get();
77
+ // Drawn at no more than the style's density, then scaled up to the view: the glow's soft
78
+ // gradients look the same at 2x with fewer pixels to fill, while the wave's line needs 3x.
79
+ const density = Math.min(1, visualStyle.maxScale / pixelRatio);
80
+ const canvasSize = useMemo<VoiceCanvasSize>(
81
+ () => ({
82
+ height: layout.height * density,
83
+ pixelRatio: pixelRatio * density,
84
+ width: layout.width * density,
85
+ }),
86
+ [density, layout.height, layout.width, pixelRatio],
87
+ );
88
+ const hitWidth = visualStyle.hitArea.width * side;
89
+ const hitHeight = visualStyle.hitArea.height * side;
90
+
91
+ /** A touch on the hit area, centred on the view and normalized to its shorter side, y up. */
92
+ const toLocalPoint = useCallback(
93
+ (locationX: number, locationY: number) => ({
94
+ x: (locationX - hitWidth / 2) / (side || 1),
95
+ y: (hitHeight / 2 - locationY) / (side || 1),
96
+ }),
97
+ [hitHeight, hitWidth, side],
98
+ );
99
+
100
+ const handleTouchEnd = useCallback(
101
+ (event: GestureResponderEvent) => {
102
+ const { x, y } = toLocalPoint(event.nativeEvent.locationX, event.nativeEvent.locationY);
103
+ if (visualStyle.contains(x, y)) tap(x, y);
104
+ },
105
+ [tap, toLocalPoint, visualStyle],
106
+ );
107
+
108
+ const handlePointerMove = useCallback(
109
+ (event: PointerEvent) => {
110
+ // Only a hovering mouse or pen ripples it; a finger has no hover.
111
+ const { buttons, offsetX, offsetY, pointerType } = event.nativeEvent;
112
+ if (pointerType === 'touch') return;
113
+ const { x, y } = toLocalPoint(offsetX, offsetY);
114
+ const hovering = buttons === 0 && visualStyle.contains(x, y);
115
+ hover(hovering ? x : null, hovering ? y : null);
116
+ },
117
+ [hover, toLocalPoint, visualStyle],
118
+ );
119
+
120
+ const handlePointerLeave = useCallback(() => hover(null, null), [hover]);
121
+
122
+ return (
123
+ <View
124
+ accessibilityLabel={accessibilityLabel}
125
+ accessibilityRole='image'
126
+ accessibilityValue={
127
+ accessibilityValue ?? { text: stateLabels?.[state] ?? DEFAULT_STATE_LABELS[state] }
128
+ }
129
+ accessible
130
+ onLayout={handleLayout}
131
+ pointerEvents={pointerEvents}
132
+ style={style}
133
+ {...restProps}
134
+ >
135
+ {side > 0 ? (
136
+ <>
137
+ <Animated.View
138
+ pointerEvents='none'
139
+ style={[
140
+ styles.canvas,
141
+ {
142
+ height: canvasSize.height,
143
+ left: (layout.width - canvasSize.width) / 2,
144
+ opacity,
145
+ top: (layout.height - canvasSize.height) / 2,
146
+ transform: [{ scale: 1 / density }],
147
+ width: canvasSize.width,
148
+ },
149
+ ]}
150
+ >
151
+ {renderCanvas(visualization, canvasSize)}
152
+ </Animated.View>
153
+ <View
154
+ // A transparent view with only handlers could be flattened away, leaving no target.
155
+ collapsable={false}
156
+ onPointerLeave={handlePointerLeave}
157
+ onPointerMove={handlePointerMove}
158
+ onTouchEnd={handleTouchEnd}
159
+ style={[
160
+ styles.hitArea,
161
+ {
162
+ height: hitHeight,
163
+ left: (layout.width - hitWidth) / 2,
164
+ top: (layout.height - hitHeight) / 2,
165
+ width: hitWidth,
166
+ },
167
+ ]}
168
+ testID='voice-visualization-hit-area'
169
+ />
170
+ </>
171
+ ) : null}
172
+ </View>
173
+ );
174
+ };
175
+
176
+ const styles = StyleSheet.create({
177
+ canvas: { position: 'absolute' },
178
+ hitArea: { position: 'absolute' },
179
+ });