stream-chat-react-native-core 9.10.2-beta.4 → 9.11.0-beta.1

package/src/ai-components/voice-visualization/palette.ts

@@ -0,0 +1,59 @@
+import { processColor } from 'react-native';
+
+import type { Hsv } from './types';
+
+export const MAXIMUM_COLORS = 5;
+
+export type Rgb = [number, number, number];
+
+/** Used when neither the `colors` prop nor the theme gives a readable color. */
+export const FALLBACK_COLOR = '#005fff';
+
+/**
+ * Reads any color React Native understands (names, `#rgb[a]`, `#rrggbb[aa]`, `rgb()`, `hsl()`, ...)
+ * into sRGB channels from 0 to 1, or null when it cannot be read. Platform colors have no
+ * channels to read and count as unreadable.
+ */
+export const parseColor = (color: string): Rgb | null => {
+  if (typeof color !== 'string' || !color.trim()) return null;
+  const processed = processColor(color.trim());
+  if (typeof processed !== 'number') return null;
+  // 0xAARRGGBB, signed on Android.
+  const argb = processed < 0 ? processed + 2 ** 32 : processed;
+  const channel = (shift: number) => (Math.floor(argb / 2 ** shift) % 256) / 255;
+  return [channel(16), channel(8), channel(0)];
+};
+
+/** RGB channels from 0 to 1 as HSV, keeping full saturation and value so muted colors render true. */
+export const rgbToHsv = ([r, g, b]: Rgb): Hsv => {
+  const high = Math.max(r, g, b);
+  const low = Math.min(r, g, b);
+  const delta = high - low;
+  let hue = 0;
+  if (delta !== 0) {
+    if (high === r) hue = ((((g - b) / delta) % 6) + 6) % 6;
+    else if (high === g) hue = (b - r) / delta + 2;
+    else hue = (r - g) / delta + 4;
+  }
+  return {
+    hue: (((hue / 6) % 1) + 1) % 1,
+    saturation: high === 0 ? 0 : Math.min(Math.max(delta / high, 0), 1),
+    value: Math.min(Math.max(high, 0), 1),
+  };
+};
+
+/** Colors as the HSV the renderers read, at most five; colors that cannot be read are skipped. */
+export const toHsvPalette = (colors: readonly string[]): Hsv[] =>
+  colors
+    .map(parseColor)
+    .filter((rgb): rgb is Rgb => rgb !== null)
+    .slice(0, MAXIMUM_COLORS)
+    .map(rgbToHsv);
+
+/** Relative luminance of a color, or null when it cannot be read. */
+export const luminance = (color: string) => {
+  const rgb = parseColor(color);
+  if (!rgb) return null;
+  const [r, g, b] = rgb.map((c) => (c <= 0.03928 ? c / 12.92 : Math.pow((c + 0.055) / 1.055, 2.4)));
+  return 0.2126 * r + 0.7152 * g + 0.0722 * b;
+};

package/src/ai-components/voice-visualization/simplex.ts

@@ -0,0 +1,92 @@
+/**
+ * 3-D simplex noise, a port of the glow shader's `snoise` (Ashima Arts / Stefan Gustavson), so the
+ * wave drawn on the CPU moves with the same speech rhythm as the shader, as on iOS and the web.
+ *
+ * It runs in float32 like a GPU, with `mod` as a multiply by a float32 reciprocal, because the
+ * hash is sensitive to rounding: in exact arithmetic some inputs hash to a neighbouring gradient.
+ * Every function is a worklet, as the motion runs on the UI thread.
+ */
+
+const INV_289 = Math.fround(1 / 289);
+const ONE_THIRD = Math.fround(1 / 3);
+const ONE_SIXTH = Math.fround(1 / 6);
+const NS_X = Math.fround(2 / 7);
+const NS_Y = Math.fround(Math.fround(0.5 * Math.fround(1 / 7)) - 1);
+const NS_Z = Math.fround(1 / 7);
+
+const mod289 = (x: number) => {
+  'worklet';
+  const f = Math.fround;
+  return f(x - f(289 * Math.floor(f(x * INV_289))));
+};
+
+const permute = (x: number) => {
+  'worklet';
+  const f = Math.fround;
+  return mod289(f(f(f(x * 34) + 1) * x));
+};
+
+const taylorInvSqrt = (r: number) => {
+  'worklet';
+  const f = Math.fround;
+  return f(1.79284291400159 - f(0.85373472095314 * r));
+};
+
+/** A corner's contribution, its gradient from its hash as the shader's x_/y_/h/sh steps compute it. */
+const corner = (p: number, x: number, y: number, z: number) => {
+  'worklet';
+  const f = Math.fround;
+  const j = f(p - f(49 * Math.floor(f(f(p * NS_Z) * NS_Z))));
+  const xi = Math.floor(f(j * NS_Z));
+  const yi = Math.floor(f(j - f(7 * xi)));
+  const gx0 = f(f(xi * NS_X) + NS_Y);
+  const gy0 = f(f(yi * NS_X) + NS_Y);
+  const gz = f(f(1 - Math.abs(gx0)) - Math.abs(gy0));
+  const sh = gz <= 0 ? -1 : 0;
+  const gx = f(gx0 + f((Math.floor(gx0) * 2 + 1) * sh));
+  const gy = f(gy0 + f((Math.floor(gy0) * 2 + 1) * sh));
+  const norm = taylorInvSqrt(gx * gx + gy * gy + gz * gz);
+  const m = Math.max(0.6 - (x * x + y * y + z * z), 0);
+  return m * m * m * m * (gx * norm * x + gy * norm * y + gz * norm * z);
+};
+
+/** Simplex noise at (x, y, z), from about -1 to 1. */
+export const snoise = (inX: number, inY: number, inZ: number) => {
+  'worklet';
+  const f = Math.fround;
+  const vx = f(inX);
+  const vy = f(inY);
+  const vz = f(inZ);
+  const s = f(f(vx + vy + vz) * ONE_THIRD);
+  const ix = Math.floor(f(vx + s));
+  const iy = Math.floor(f(vy + s));
+  const iz = Math.floor(f(vz + s));
+  const t = f(f(ix + iy + iz) * ONE_SIXTH);
+  const x0 = f(f(vx - ix) + t);
+  const y0 = f(f(vy - iy) + t);
+  const z0 = f(f(vz - iz) + t);
+
+  const gx = x0 >= y0 ? 1 : 0;
+  const gy = y0 >= z0 ? 1 : 0;
+  const gz = z0 >= x0 ? 1 : 0;
+  const i1x = Math.min(gx, 1 - gz);
+  const i1y = Math.min(gy, 1 - gx);
+  const i1z = Math.min(gz, 1 - gy);
+  const i2x = Math.max(gx, 1 - gz);
+  const i2y = Math.max(gy, 1 - gx);
+  const i2z = Math.max(gz, 1 - gy);
+
+  const mx = mod289(ix);
+  const my = mod289(iy);
+  const mz = mod289(iz);
+  const hash = (dx: number, dy: number, dz: number) =>
+    permute(f(f(permute(f(f(permute(f(mz + dz)) + my) + dy)) + mx) + dx));
+
+  return (
+    42 *
+    (corner(hash(0, 0, 0), x0, y0, z0) +
+      corner(hash(i1x, i1y, i1z), x0 - i1x + 1 / 6, y0 - i1y + 1 / 6, z0 - i1z + 1 / 6) +
+      corner(hash(i2x, i2y, i2z), x0 - i2x + 1 / 3, y0 - i2y + 1 / 3, z0 - i2z + 1 / 3) +
+      corner(hash(1, 1, 1), x0 - 0.5, y0 - 0.5, z0 - 0.5))
+  );
+};

package/src/ai-components/voice-visualization/styles.ts

@@ -0,0 +1,42 @@
+import { LEVEL_RESPONSE, type LevelResponse } from './driver';
+
+/**
+ * What tells the glow and the wave apart besides how they draw, as on iOS.
+ *
+ * - `levelResponse`: the wave's height tracks the level directly, so it follows it smoothly;
+ *   per-syllable jumps would make it jitter. The glow follows it closely.
+ * - `maxScale`: the densest it renders. The glow is soft all over, so 2x is enough; the wave's
+ *   thin line is drawn at the screen's own scale, up to 3x, to stay crisp.
+ * - `contains`: whether a point, centred and normalized to the shorter side with y up, is on the
+ *   shape rather than the empty space around it.
+ */
+export type VoiceVisualizationStyle = {
+  contains: (x: number, y: number) => boolean;
+  /** The shape's bounds as a share of the shorter side, for the area that takes touches. */
+  hitArea: { width: number; height: number };
+  levelResponse: LevelResponse;
+  maxScale: number;
+};
+
+/** How far from the centre a tap still lands on the globe, as a share of the shorter side. */
+const GLOBE_REACH = 0.32;
+
+export const GLOW_STYLE: VoiceVisualizationStyle = {
+  contains: (x, y) => {
+    'worklet';
+    return Math.hypot(x, y) < GLOBE_REACH;
+  },
+  hitArea: { height: 2 * GLOBE_REACH, width: 2 * GLOBE_REACH },
+  levelResponse: LEVEL_RESPONSE.steady,
+  maxScale: 2,
+};
+
+export const WAVE_STYLE: VoiceVisualizationStyle = {
+  contains: (x, y) => {
+    'worklet';
+    return Math.abs(x) < 0.5 && Math.abs(y) < 0.14;
+  },
+  hitArea: { height: 0.28, width: 1 },
+  levelResponse: LEVEL_RESPONSE.smooth,
+  maxScale: 3,
+};

package/src/ai-components/voice-visualization/types.ts

@@ -0,0 +1,44 @@
+import type { ViewProps } from 'react-native';
+
+import type { SharedValue } from 'react-native-reanimated';
+
+/**
+ * What a voice agent is doing, which a voice visualization shows as motion.
+ *
+ * The states differ only in how a visualization moves, never in its colors, and changing the
+ * state eases from one motion into the next.
+ *
+ * - `idle`: waiting to be spoken to, a calm, barely-there shimmer.
+ * - `connecting`: not ready yet, a loader sweep with a pulsing breath.
+ * - `listening`: hearing somebody, a gentle expand and contract.
+ * - `thinking`: working on an answer, a slow turn or a travelling wave.
+ * - `speaking`: talking, strong motion that ebbs and pauses like speech.
+ */
+export type VoiceAgentState = 'idle' | 'connecting' | 'listening' | 'thinking' | 'speaking';
+
+export type VoiceVisualizationProps = Omit<ViewProps, 'children'> & {
+  /** What the agent is doing, which sets how the visualization moves. */
+  state: VoiceAgentState;
+  /**
+   * The level of whoever is talking, from 0 to 1, read once a frame on the UI thread. Feed it at
+   * 15 Hz or more for the visualization to move with the words; `null` animates speech on its own.
+   * See `useDictationAudioLevel` for the level of the microphone while dictating.
+   */
+  audioLevel?: SharedValue<number | null>;
+  /**
+   * One to five colors. Past five are ignored, and colors that cannot be read are skipped. When
+   * omitted or empty, the theme's `voiceVisualization.colors` are used.
+   */
+  colors?: string[];
+  /** From 0 to 2: how far and how fast it moves. Colors never change with it. Defaults to 1. */
+  expressivity?: number;
+  /**
+   * What screen readers read as the value for each state, for example to translate them. Defaults
+   * to "Ready", "Connecting", "Listening", "Thinking" and "Speaking". The label, "Voice agent", is
+   * `accessibilityLabel`.
+   */
+  stateLabels?: Partial<Record<VoiceAgentState, string>>;
+};
+
+/** A color as hue (a fraction of a turn), saturation and value, each from 0 to 1. */
+export type Hsv = { hue: number; saturation: number; value: number };

package/src/ai-components/voice-visualization/useReduceMotion.ts

@@ -0,0 +1,34 @@
+import { useEffect, useState } from 'react';
+import { AccessibilityInfo } from 'react-native';
+
+/**
+ * Whether the system asks for less motion: Reduce Motion on iOS, animations off on Android. It
+ * follows the setting while mounted.
+ *
+ * The AI entry point cannot reach the core SDK's own hook, which also only applies once its
+ * accessibility layer is enabled; the voice visualizations always honour the setting, as on iOS
+ * and Android.
+ */
+export const useReduceMotion = () => {
+  const [reduceMotion, setReduceMotion] = useState(false);
+
+  useEffect(() => {
+    let mounted = true;
+    AccessibilityInfo.isReduceMotionEnabled?.()
+      .then((enabled) => {
+        // It starts off; only turning it on needs a render.
+        if (mounted && enabled) setReduceMotion(true);
+      })
+      .catch(() => undefined);
+    const subscription = AccessibilityInfo.addEventListener?.(
+      'reduceMotionChanged',
+      setReduceMotion,
+    );
+    return () => {
+      mounted = false;
+      subscription?.remove();
+    };
+  }, []);
+
+  return reduceMotion;
+};

package/src/ai-components/voice-visualization/useVoiceVisualization.ts

@@ -0,0 +1,153 @@
+import { useCallback, useEffect, useMemo, useState } from 'react';
+import { useColorScheme } from 'react-native';
+
+import {
+  type FrameInfo,
+  runOnUI,
+  useFrameCallback,
+  useSharedValue,
+  withTiming,
+} from 'react-native-reanimated';
+
+import {
+  advanceDriver,
+  createDriverState,
+  hoverDriver,
+  tapDriver,
+  type VoiceDriverInput,
+} from './driver';
+import { audioLevelInput, expressivityInput } from './math';
+import { FALLBACK_COLOR, luminance, toHsvPalette } from './palette';
+import type { VoiceVisualizationStyle } from './styles';
+import type { VoiceVisualizationProps } from './types';
+import { useReduceMotion } from './useReduceMotion';
+
+import { useTheme } from '../contexts';
+
+/** How long the visualization takes to fade in once its first frame is drawn, as on iOS. */
+const FADE_IN_MS = 300;
+
+/** Whether to draw for a dark background: the theme's backdrop when readable, else the scheme. */
+const useDark = (backdropColor: string | undefined) => {
+  const scheme = useColorScheme();
+  return useMemo(() => {
+    const backdrop = backdropColor ? luminance(backdropColor) : null;
+    if (backdrop !== null) return backdrop < 0.5 ? 1 : 0;
+    return scheme === 'dark' ? 1 : 0;
+  }, [backdropColor, scheme]);
+};
+
+/**
+ * A voice visualization's motion, advanced once a frame on the UI thread.
+ *
+ * The driver lives in a shared value the renderers derive from, so a frame redraws without
+ * re-rendering React, and `audioLevel` is read in the frame callback, so a level changing many
+ * times a second re-renders nothing either.
+ */
+export const useVoiceVisualization = (
+  { audioLevel, colors, expressivity, state }: VoiceVisualizationProps,
+  style: VoiceVisualizationStyle,
+) => {
+  const { theme } = useTheme();
+  const reduceMotion = useReduceMotion();
+  const themeColors = theme.voiceVisualization?.colors;
+  const dark = useDark(theme.voiceVisualization?.backdropColor);
+
+  const colorsKey = (colors ?? []).join('|');
+  const themeKey = (themeColors ?? []).join('|');
+  const palette = useMemo(() => {
+    const own = toHsvPalette(colors ?? []);
+    if (own.length) return own;
+    const fromTheme = toHsvPalette(themeColors ?? []);
+    return fromTheme.length ? fromTheme : toHsvPalette([FALLBACK_COLOR]);
+    // Keyed by the colors themselves, so a new array of the same colors changes nothing.
+    // eslint-disable-next-line react-hooks/exhaustive-deps
+  }, [colorsKey, themeKey]);
+
+  const input: VoiceDriverInput = {
+    audioLevel: null,
+    colors: palette,
+    expressivity: expressivityInput(expressivity),
+    reduceMotion,
+    state,
+  };
+
+  // Starts settled on the first input, so the first frame shows it rather than easing into it.
+  const [initialDriver] = useState(() => createDriverState(input, style.levelResponse));
+  const driver = useSharedValue(initialDriver);
+  const inputValue = useSharedValue(input);
+  const darkValue = useSharedValue(dark);
+  const opacity = useSharedValue(0);
+  const fadingIn = useSharedValue(false);
+
+  useEffect(() => {
+    inputValue.value = {
+      audioLevel: null,
+      colors: palette,
+      expressivity: expressivityInput(expressivity),
+      reduceMotion,
+      state,
+    };
+  }, [expressivity, inputValue, palette, reduceMotion, state]);
+
+  useEffect(() => {
+    darkValue.value = dark;
+  }, [dark, darkValue]);
+
+  const onFrame = useCallback(
+    (frame: FrameInfo) => {
+      'worklet';
+      const seconds = (frame.timeSincePreviousFrame ?? 0) / 1000;
+      const current = inputValue.value;
+      const level = audioLevelInput(audioLevel ? audioLevel.value : null);
+      const next = level === current.audioLevel ? current : { ...current, audioLevel: level };
+      driver.modify((d) => {
+        'worklet';
+        advanceDriver(d, next, seconds);
+        return d;
+      });
+      // It fades in once it has something to show, rather than flashing empty while the first
+      // frame lands.
+      if (!fadingIn.value) {
+        fadingIn.value = true;
+        opacity.value = withTiming(1, { duration: FADE_IN_MS });
+      }
+    },
+    [audioLevel, driver, fadingIn, inputValue, opacity],
+  );
+  useFrameCallback(onFrame);
+
+  /** A tap at a point centred and normalized to the shorter side, y up. */
+  const tap = useCallback(
+    (x: number, y: number) => {
+      runOnUI((tapX: number, tapY: number) => {
+        'worklet';
+        driver.modify((d) => {
+          'worklet';
+          tapDriver(d, inputValue.value, tapX, tapY);
+          return d;
+        });
+      })(x, y);
+    },
+    [driver, inputValue],
+  );
+
+  /** Where a pointer hovers, or null once it has left. */
+  const hover = useCallback(
+    (x: number | null, y: number | null) => {
+      runOnUI((hoverX: number | null, hoverY: number | null) => {
+        'worklet';
+        driver.modify((d) => {
+          'worklet';
+          hoverDriver(d, inputValue.value, hoverX, hoverY);
+          return d;
+        });
+      })(x, y);
+    },
+    [driver, inputValue],
+  );
+
+  return { dark: darkValue, driver, hover, opacity, tap };
+};
+
+export type VoiceVisualization = ReturnType<typeof useVoiceVisualization>;

package/src/ai-components/voice-visualization/waveGeometry.ts

@@ -0,0 +1,137 @@
+import { Skia, type SkPath } from '@shopify/react-native-skia';
+
+import { driverReact, driverScale, type VoiceDriverState } from './driver';
+import { clamp, mix, saturate, smoothstep, speechEnv, vivid } from './math';
+
+const POINTS = 160;
+export const WAVE_STOPS = 13;
+export const WAVE_START = -0.42;
+export const WAVE_END = 0.42;
+
+/** The stops' positions along the gradient, evenly spaced. */
+export const WAVE_POSITIONS = Array.from({ length: WAVE_STOPS }, (_, s) => s / (WAVE_STOPS - 1));
+
+const sq = (x: number) => {
+  'worklet';
+  return x * x;
+};
+
+const bandCenter = (i: number, seg: number, t: number) => {
+  'worklet';
+  switch (i) {
+    case 0:
+      return 0.5 * seg + 0.12 * Math.sin(t * 0.13);
+    case 1:
+      return 1.5 * seg + 0.13 * Math.sin(t * 0.11 + 2.1);
+    case 2:
+      return 2.5 * seg + 0.12 * Math.sin(t * 0.17 + 4.2);
+    case 3:
+      return 3.5 * seg + 0.12 * Math.sin(t * 0.15 + 1);
+    default:
+      return 4.5 * seg + 0.12 * Math.sin(t * 0.1 + 3.3);
+  }
+};
+
+/** The string's height above its centre line at each point, centred and normalized, y up. */
+export const waveHeights = (d: VoiceDriverState) => {
+  'worklet';
+  const t = d.time;
+  const react = driverReact(d);
+  const center = d.load * 0.3 * Math.sin(t * 1.3);
+  const freq = 24 + 26 * react + 10 * d.flow;
+  const speak = smoothstep(0.6, 1, react);
+  const drive = mix(speak * 0.26 * speechEnv(t), 0.3 * d.mic, d.voice);
+  const reach = 0.018 + 0.022 * react + drive + 0.14 * d.flow + 0.1 * d.load;
+  const tapAlive = d.tapAge < 1.5;
+  const hovered = d.hoverAmount > 0.001;
+  const xs: number[] = [];
+  const ys: number[] = [];
+  for (let i = 0; i <= POINTS; i++) {
+    const x = WAVE_START + ((WAVE_END - WAVE_START) * i) / POINTS;
+    let y = Math.exp(-sq((x - center) / 0.18)) * reach * Math.sin(x * freq - t * 5);
+    if (tapAlive) {
+      y +=
+        Math.exp(-sq((x - d.tapX) / 0.12)) *
+        Math.sin((x - d.tapX) * 40 - d.tapAge * 24) *
+        Math.exp(-d.tapAge * 2.6) *
+        0.05;
+    }
+    if (hovered) {
+      y +=
+        Math.exp(-sq((x - d.hoverX) / 0.12)) *
+        Math.sin((x - d.hoverX) * 40 - t * 7) *
+        d.hoverAmount *
+        0.04;
+    }
+    xs.push(x);
+    ys.push(y);
+  }
+  return { xs, ys };
+};
+
+type LegacyPath = SkPath & {
+  lineTo: (x: number, y: number) => void;
+  moveTo: (x: number, y: number) => void;
+};
+
+/**
+ * The wave as a path in a `width` by `height` canvas. Newer Skia versions build immutable paths with
+ * `PathBuilder`; earlier 2.x versions mutate `SkPath` itself.
+ */
+export const wavePath = (d: VoiceDriverState, width: number, height: number): SkPath => {
+  'worklet';
+  const m = Math.min(width, height);
+  const scale = driverScale(d);
+  const { xs, ys } = waveHeights(d);
+  const px = (i: number) => width / 2 + xs[i] * m * scale;
+  const py = (i: number) => height / 2 - ys[i] * m * scale;
+  if (Skia.PathBuilder) {
+    const builder = Skia.PathBuilder.Make();
+    builder.moveTo(px(0), py(0));
+    for (let i = 1; i < xs.length; i++) builder.lineTo(px(i), py(i));
+    return builder.build();
+  }
+  const path = Skia.Path.Make() as LegacyPath;
+  path.moveTo(px(0), py(0));
+  for (let i = 1; i < xs.length; i++) path.lineTo(px(i), py(i));
+  return path;
+};
+
+/**
+ * The string's colors at its gradient stops, as RGBA from 0 to 1: up to five bands drifting along
+ * it, weighed by how near their centres are, its ends fading out.
+ */
+export const waveColors = (d: VoiceDriverState, dark: number) => {
+  'worklet';
+  const band: number[][] = [];
+  for (let i = 0; i < 5; i++) {
+    const rgb = [0, 0, 0];
+    vivid(d.colors[i], dark, rgb);
+    band.push(rgb);
+  }
+  const seg = 1 / Math.max(d.count, 1);
+  const bandWidth = seg * 0.7;
+  const stops: Float32Array[] = [];
+  for (let s = 0; s < WAVE_STOPS; s++) {
+    const x = WAVE_START + ((WAVE_END - WAVE_START) * s) / (WAVE_STOPS - 1);
+    const gx = clamp(x + 0.5, 0, 1);
+    const color = [0, 0, 0];
+    let total = 1e-4;
+    for (let i = 0; i < 5; i++) {
+      const present = i === 0 ? 1 : clamp(d.count - i, 0, 1);
+      const weight = Math.exp(-sq((gx - bandCenter(i, seg, d.time)) / bandWidth)) * present;
+      for (let j = 0; j < 3; j++) color[j] += band[i][j] * weight;
+      total += weight;
+    }
+    for (let j = 0; j < 3; j++) color[j] /= total;
+    saturate(color, 1.2);
+    saturate(color, d.sat);
+    const alpha = clamp(smoothstep(0.4, 0.24, Math.abs(x)) * d.bright, 0, 1);
+    stops.push(new Float32Array([color[0], color[1], color[2], alpha]));
+  }
+  return stops;
+};
+
+/** The line is 0.6% of the shorter side, and never under 1.5 device pixels, as on Android and iOS. */
+export const waveStrokeWidth = (width: number, height: number, pixelRatio: number) =>
+  Math.max(0.006 * Math.min(width, height), 1.5 / pixelRatio);

package/src/version.json

@@ -1,3 +1,3 @@
 {
-  "version": "9.10.2-beta.4"
+  "version": "9.11.0-beta.1"
 }