realtime-avatar 0.30.0 → 0.32.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
package/dist/nextjs.js CHANGED
@@ -1,5 +1,5 @@
1
- import { createProxyHandler } from './chunk-GLIEEJMW.js';
2
- import './chunk-WXVBPMDN.js';
1
+ import { createProxyHandler } from './chunk-NDCZABOK.js';
2
+ import './chunk-NKS5ZIWN.js';
3
3
  import './chunk-MCIXU6XP.js';
4
4
 
5
5
  // ../proxy/src/nextjs.ts
@@ -2,11 +2,12 @@ import { LiveKitRoomProps } from '@livekit/react-native';
2
2
  export { AudioSession, VideoTrack, VideoTrackProps, registerGlobals } from '@livekit/react-native';
3
3
  import { L as LiveKitSessionGrant, A as AvatarVideoFit } from './proxy-client-BlaFxR6S.js';
4
4
  export { a as AdaptivePlayoutController, b as AdaptivePlayoutDecision, c as AdaptivePlayoutOptions, d as AdaptivePlayoutSample, e as ApproachingEndEvent, f as ApproachingEndReason, g as AvatarConnectionDetails, h as AvatarSessionClient, B as BehaviorSnapshot, C as CallTranscript, i as CallTranscriptSegment, j as CartesiaTtsModel, k as ClipResult, l as ClosingTurnResult, m as CreditsLowEvent, D as DEFAULT_APPROACHING_END_LEAD_SECONDS, n as DEFAULT_CREDITS_LOW_LEAD_SECONDS, o as DEFAULT_FISH_TTS_MODEL, p as DEFAULT_GOVERNOR_CONFIG, q as DEFAULT_GRACE_CEILING_SECONDS, r as DEFAULT_GRACE_WINDOW_LEAD_SECONDS, s as DEFAULT_IDLE_SECONDS, t as DEFAULT_IDLE_WARN_LEAD_SECONDS, u as DEFAULT_READY_TIMEOUT_SECONDS, v as DEFAULT_TURN_TIMEOUT_SECONDS, w as DeclaredInputSource, E as EndReason, x as EndedEvent, y as ExtendResult, F as FishTtsModel, z as FreezeReadingFn, G as Governor, H as GovernorAction, I as GovernorConfig, J as GovernorSignal, K as GovernorState, M as GraceWindowClosedEvent, N as GraceWindowOpenEvent, O as GraceWindowState, P as IdleWarningEvent, Q as InboundRtpCursor, R as InboundRtpReading, S as KnownBehaviorState, T as LLMProvider, U as LLMSelection, V as LiveKitAvatarGrantState, W as LiveKitAvatarGrantStatus, X as LiveKitCapacityState, Y as LiveKitConnectionStatus, Z as LiveKitSessionRequest, _ as LiveKitSessionStartResult, $ as MAX_RECONNECT_ATTEMPTS, a0 as MAX_SESSION_INSTRUCTIONS_CHARS, a1 as ProxyClientOptions, a2 as QualityCap, a3 as RECONNECT_BACKOFF_MS, a4 as RealtimeAvatarRequestOptions, a5 as RealtimeSessionApi, a6 as RealtimeSessionMedia, a7 as RealtimeSessionRoomSinks, a8 as ReconnectPolicy, a9 as ReconnectingEvent, aa as RecoveryState, ab as RetryStep, ac as SendTurnOptions, ad as SessionBehavior, ae as SessionClip, af as SessionClocks, ag as SessionEndReason, ah as SessionLifecycleApi, ai as SessionLifecyclePhase, aj as SessionLifecyclePhaseKind, ak as SessionLifecycleRoomBridge, al as SessionLifecycleRoomBridgeProps, am as SessionReadinessFailure, an as SurfaceLayers, ao as TranscriptSender, ap as TranscriptSenderOptions, aq as TurnState, ar as TurnTimeoutEvent, as as UseAvatarQualityGovernorInput, at as UseLiveKitAvatarGrantInput, au as UseRealtimeSessionInput, av as UseSessionLifecycleInput, aw as VoiceError, ax as VoiceErrorCode, ay as VoiceSpec, az as VoiceSpecInput, aA as capacityErrorFromBusy, aB as capacityStateFromGrant, aC as createProxyClient, aD as fishTtsModelSchema, aE as isNativeLiveTrackSubscribed, aF as knownBehaviorStates, aG as mapTurnState, aH as readInboundRtp, aI as sessionBehaviorSchema, aJ as sessionClipSchema, aK as splitCallTranscript, aL as useAvatarAdaptivePlayoutDelay, aM as useAvatarCamera, aN as useAvatarPlayoutDelay, aO as useAvatarQualityGovernor, aP as useCallTranscript, aQ as useLiveKitAvatarGrant, aR as useLiveTrackProducing, aS as useMicLease, aT as useRealtimeSession, aU as useReleaseMicLeaseOnTrackEnded, aV as useSessionLifecycle } from './proxy-client-BlaFxR6S.js';
5
+ import { Room } from 'livekit-client';
6
+ export { DisconnectReason, RemoteAudioTrack, RemoteTrack, RemoteVideoTrack, Room, RoomEvent, SendTextOptions, Track } from 'livekit-client';
5
7
  import { ReactNode, ReactElement } from 'react';
6
8
  import { StyleProp, ViewStyle } from 'react-native';
7
9
  export { useChat, useConnectionState, useLocalParticipant, useRoomContext, useTrackToggle, useTranscriptions, useVoiceAssistant } from '@livekit/components-react';
8
10
  export { D as DEFAULT_AVATAR_PLAYOUT_DELAY_SECONDS } from './playout-delay-CtRZdtsj.js';
9
- export { DisconnectReason, RemoteAudioTrack, RemoteTrack, RemoteVideoTrack, Room, RoomEvent, SendTextOptions, Track } from 'livekit-client';
10
11
  import 'zod';
11
12
 
12
13
  type RealtimeAvatarLiveKitRoomProps = Omit<LiveKitRoomProps, "serverUrl" | "token" | "connect" | "audio" | "video" | "options"> & {
@@ -32,6 +33,12 @@ type RealtimeAvatarLiveKitRoomProps = Omit<LiveKitRoomProps, "serverUrl" | "toke
32
33
  * LiveKitRoom props) can still one-line the session lifecycle.
33
34
  */
34
35
  declare function useRealtimeAvatarAudioSession(enabled: boolean): void;
36
+ /**
37
+ * Run `clearStaleInputMute` each time `room`'s local microphone goes live: when it is published
38
+ * unmuted and when it is unmuted later, for as long as that publication stays published and
39
+ * unmuted and this hook stays mounted.
40
+ */
41
+ declare function useClearStaleInputMute(room: Room | undefined): void;
35
42
  /** Thin typed bridge from a Realtime Avatar grant into the native LiveKitRoom. */
36
43
  declare function RealtimeAvatarLiveKitRoom(props: RealtimeAvatarLiveKitRoomProps): ReactElement;
37
44
 
@@ -126,4 +133,4 @@ type AvatarVideoSurfaceProps = {
126
133
  */
127
134
  declare function AvatarVideoSurface(props: AvatarVideoSurfaceProps): ReactElement;
128
135
 
129
- export { AvatarVideoFit, AvatarVideoSurface, type AvatarVideoSurfaceProps, type IdleVideoRender, RealtimeAvatarLiveKitRoom, type RealtimeAvatarLiveKitRoomProps, useRealtimeAvatarAudioSession };
136
+ export { AvatarVideoFit, AvatarVideoSurface, type AvatarVideoSurfaceProps, type IdleVideoRender, RealtimeAvatarLiveKitRoom, type RealtimeAvatarLiveKitRoomProps, useClearStaleInputMute, useRealtimeAvatarAudioSession };
@@ -3,8 +3,9 @@ export { AudioSession, VideoTrack, registerGlobals } from '@livekit/react-native
3
3
  import { useState, useRef, useEffect, useCallback, createElement, useMemo } from 'react';
4
4
  import { useLocalParticipant, useConnectionState, useTranscriptions, useMaybeRoomContext, useVoiceAssistant, isTrackReference, useRoomContext, useChat, useAudioPlayback } from '@livekit/components-react';
5
5
  export { useChat, useConnectionState, useLocalParticipant, useRoomContext, useTrackToggle, useTranscriptions, useVoiceAssistant } from '@livekit/components-react';
6
- import { ConnectionState, isRemoteTrack, VideoQuality, ConnectionQuality, RoomEvent, TrackEvent, Track, ConnectionError, ConnectionErrorReason, ParticipantEvent, DisconnectReason, createLocalVideoTrack, Room } from 'livekit-client';
6
+ import { ConnectionState, RoomEvent, Track, isRemoteTrack, VideoQuality, ConnectionQuality, TrackEvent, ConnectionError, ConnectionErrorReason, ParticipantEvent, DisconnectReason, createLocalVideoTrack, Room } from 'livekit-client';
7
7
  export { DisconnectReason, Room, RoomEvent, Track } from 'livekit-client';
8
+ import { AudioDeviceModule, AudioEngineMuteMode } from '@livekit/react-native-webrtc';
8
9
  import { Platform, StyleSheet, Animated, Image, View, Text } from 'react-native';
9
10
  import { z } from 'zod';
10
11
 
@@ -159,6 +160,51 @@ function useRealtimeAvatarAudioSession(enabled) {
159
160
  };
160
161
  }, [enabled]);
161
162
  }
163
+ var INPUT_LIVE_WAIT_MS = 5e3;
164
+ var INPUT_LIVE_POLL_MS = 100;
165
+ async function clearStaleInputMute(stillWanted) {
166
+ if (Platform.OS !== "ios") return;
167
+ try {
168
+ if (AudioDeviceModule.getMuteMode() !== AudioEngineMuteMode.VoiceProcessing) return;
169
+ for (const deadline = Date.now() + INPUT_LIVE_WAIT_MS; !AudioDeviceModule.isRecording(); ) {
170
+ if (Date.now() >= deadline || !stillWanted()) return;
171
+ await new Promise((resolve) => setTimeout(resolve, INPUT_LIVE_POLL_MS));
172
+ }
173
+ if (!stillWanted()) return;
174
+ try {
175
+ await AudioDeviceModule.setMicrophoneMuted(true);
176
+ } finally {
177
+ await AudioDeviceModule.setMicrophoneMuted(false);
178
+ }
179
+ } catch {
180
+ }
181
+ }
182
+ function useClearStaleInputMute(room) {
183
+ useEffect(() => {
184
+ if (!room) return;
185
+ let mounted = true;
186
+ const local = room.localParticipant;
187
+ const clearIfLive = (publication) => {
188
+ if (publication?.source !== Track.Source.Microphone || publication.isMuted) return;
189
+ void clearStaleInputMute(
190
+ () => mounted && !publication.isMuted && local.getTrackPublication(Track.Source.Microphone) === publication
191
+ );
192
+ };
193
+ const onUnmuted = (publication, participant) => {
194
+ if (participant === local) clearIfLive(publication);
195
+ };
196
+ room.on(RoomEvent.LocalTrackPublished, clearIfLive).on(RoomEvent.TrackUnmuted, onUnmuted);
197
+ clearIfLive(local.getTrackPublication(Track.Source.Microphone));
198
+ return () => {
199
+ mounted = false;
200
+ room.off(RoomEvent.LocalTrackPublished, clearIfLive).off(RoomEvent.TrackUnmuted, onUnmuted);
201
+ };
202
+ }, [room]);
203
+ }
204
+ function StaleInputMuteClearer() {
205
+ useClearStaleInputMute(useMaybeRoomContext());
206
+ return null;
207
+ }
162
208
  function RealtimeAvatarLiveKitRoom(props) {
163
209
  const {
164
210
  grant,
@@ -186,6 +232,7 @@ function RealtimeAvatarLiveKitRoom(props) {
186
232
  video: cameraPublishOptions(grant, video),
187
233
  options: roomOptions
188
234
  },
235
+ createElement(StaleInputMuteClearer),
189
236
  children
190
237
  );
191
238
  }
@@ -3885,4 +3932,4 @@ function createProxyClient(options) {
3885
3932
  };
3886
3933
  }
3887
3934
 
3888
- export { AdaptivePlayoutController, AvatarVideoSurface, DEFAULT_APPROACHING_END_LEAD_SECONDS, DEFAULT_AVATAR_PLAYOUT_DELAY_SECONDS, DEFAULT_CREDITS_LOW_LEAD_SECONDS, DEFAULT_FISH_TTS_MODEL, DEFAULT_GOVERNOR_CONFIG, DEFAULT_GRACE_CEILING_SECONDS, DEFAULT_GRACE_WINDOW_LEAD_SECONDS, DEFAULT_IDLE_SECONDS, DEFAULT_IDLE_WARN_LEAD_SECONDS, DEFAULT_READY_TIMEOUT_SECONDS, DEFAULT_TURN_TIMEOUT_SECONDS, MAX_RECONNECT_ATTEMPTS, MAX_SESSION_INSTRUCTIONS_CHARS, RECONNECT_BACKOFF_MS, RealtimeAvatarLiveKitRoom, SessionLifecycleRoomBridge, capacityErrorFromBusy, capacityStateFromGrant, createProxyClient, fishTtsModelSchema, isNativeLiveTrackSubscribed, knownBehaviorStates, mapTurnState, readInboundRtp, sessionBehaviorSchema, sessionClipSchema, splitCallTranscript, useAvatarAdaptivePlayoutDelay, useAvatarCamera, useAvatarPlayoutDelay, useAvatarQualityGovernor, useCallTranscript, useLiveKitAvatarGrant, useLiveTrackProducing, useMicLease, useRealtimeAvatarAudioSession, useRealtimeSession, useReleaseMicLeaseOnTrackEnded, useSessionLifecycle };
3935
+ export { AdaptivePlayoutController, AvatarVideoSurface, DEFAULT_APPROACHING_END_LEAD_SECONDS, DEFAULT_AVATAR_PLAYOUT_DELAY_SECONDS, DEFAULT_CREDITS_LOW_LEAD_SECONDS, DEFAULT_FISH_TTS_MODEL, DEFAULT_GOVERNOR_CONFIG, DEFAULT_GRACE_CEILING_SECONDS, DEFAULT_GRACE_WINDOW_LEAD_SECONDS, DEFAULT_IDLE_SECONDS, DEFAULT_IDLE_WARN_LEAD_SECONDS, DEFAULT_READY_TIMEOUT_SECONDS, DEFAULT_TURN_TIMEOUT_SECONDS, MAX_RECONNECT_ATTEMPTS, MAX_SESSION_INSTRUCTIONS_CHARS, RECONNECT_BACKOFF_MS, RealtimeAvatarLiveKitRoom, SessionLifecycleRoomBridge, capacityErrorFromBusy, capacityStateFromGrant, createProxyClient, fishTtsModelSchema, isNativeLiveTrackSubscribed, knownBehaviorStates, mapTurnState, readInboundRtp, sessionBehaviorSchema, sessionClipSchema, splitCallTranscript, useAvatarAdaptivePlayoutDelay, useAvatarCamera, useAvatarPlayoutDelay, useAvatarQualityGovernor, useCallTranscript, useClearStaleInputMute, useLiveKitAvatarGrant, useLiveTrackProducing, useMicLease, useRealtimeAvatarAudioSession, useRealtimeSession, useReleaseMicLeaseOnTrackEnded, useSessionLifecycle };
package/dist/server.d.ts CHANGED
@@ -1,4 +1,4 @@
1
- export { DEFAULT_FISH_TTS_MODEL, FishTtsModel, RealtimeAvatar, RealtimeAvatarError, RealtimeAvatarHttpError, RealtimeAvatarOptions, RealtimeAvatarWebhookError, StartCallOptions, WebhookVerificationErrorCode, clipLibraryDeclarationSchema, fishTtsModelSchema, verifyTranscript, verifyVoiceWebhook } from './index.js';
2
- export { h as Asset, g as AssetKind, A as Avatar, o as AvatarClip, c as AvatarSourceSwap, b as AvatarUpdate, p as CallConnection, C as CallMode, a as CallPolicy, q as CallQueued, r as ClipDeclaration, f as ClipLibrary, d as ClipLibraryDeclaration, s as ClipLibraryPlan, e as ClipLibraryUpdate, t as ClipSource, u as ContextMessage, n as CreditBalance, E as EndCallOptions, v as EndCallReason, w as ExternalLlm, k as ListRecordingsQuery, l as ListRecordingsResponse, i as ListSessionsOptions, L as LoopRedirect, m as RecordingAccessResponse, R as RecordingArtifact, x as RecordingArtifactStatus, y as RecordingMode, S as StartCallResult, T as TranscriptPayload, j as UsageSession, U as UsageSessionPage, z as VideoPolicy, B as VoiceWebhook, D as VoiceWebhookErrorCode, V as VoiceWebhookRequest, F as isQueued, G as listRecordingsResponseSchema, H as recordingAccessResponseSchema, I as recordingArtifactSchema, J as recordingModeSchema } from './types-Bv29uEAs.js';
1
+ export { DEFAULT_FISH_TTS_MODEL, FishTtsModel, ListRendersQuery, ListRendersResponse, RealtimeAvatar, RealtimeAvatarError, RealtimeAvatarHttpError, RealtimeAvatarOptions, RealtimeAvatarWebhookError, Render, RenderAccessResponse, RenderFailureCode, RenderPlayback, RenderStatus, ShotTiming, StartCallOptions, WebhookVerificationErrorCode, clipLibraryDeclarationSchema, fishTtsModelSchema, listRendersResponseSchema, renderAccessResponseSchema, renderSchema, verifyTranscript, verifyVoiceWebhook } from './index.js';
2
+ export { h as Asset, g as AssetKind, A as Avatar, p as AvatarClip, c as AvatarSourceSwap, b as AvatarUpdate, q as CallConnection, C as CallMode, a as CallPolicy, r as CallQueued, s as ClipDeclaration, f as ClipLibrary, d as ClipLibraryDeclaration, t as ClipLibraryPlan, e as ClipLibraryUpdate, u as ClipSource, v as ContextMessage, n as CreateRenderInput, o as CreditBalance, E as EndCallOptions, w as EndCallReason, x as ExternalLlm, k as ListRecordingsQuery, l as ListRecordingsResponse, i as ListSessionsOptions, L as LoopRedirect, m as RecordingAccessResponse, R as RecordingArtifact, y as RecordingArtifactStatus, z as RecordingMode, B as RenderShot, S as StartCallResult, T as TranscriptPayload, j as UsageSession, U as UsageSessionPage, D as VideoPolicy, F as VoiceWebhook, G as VoiceWebhookErrorCode, V as VoiceWebhookRequest, H as isQueued, I as listRecordingsResponseSchema, J as recordingAccessResponseSchema, K as recordingArtifactSchema, M as recordingModeSchema } from './types-BSSqsu_Z.js';
3
3
  import './connection-history.js';
4
4
  import 'zod';
package/dist/server.js CHANGED
@@ -1,2 +1,2 @@
1
- export { DEFAULT_FISH_TTS_MODEL, RealtimeAvatar, RealtimeAvatarError, RealtimeAvatarHttpError, RealtimeAvatarWebhookError, clipLibraryDeclarationSchema, fishTtsModelSchema, isQueued, listRecordingsResponseSchema, recordingAccessResponseSchema, recordingArtifactSchema, recordingModeSchema, verifyTranscript, verifyVoiceWebhook } from './chunk-WXVBPMDN.js';
1
+ export { DEFAULT_FISH_TTS_MODEL, RealtimeAvatar, RealtimeAvatarError, RealtimeAvatarHttpError, RealtimeAvatarWebhookError, clipLibraryDeclarationSchema, fishTtsModelSchema, isQueued, listRecordingsResponseSchema, listRendersResponseSchema, recordingAccessResponseSchema, recordingArtifactSchema, recordingModeSchema, renderAccessResponseSchema, renderSchema, verifyTranscript, verifyVoiceWebhook } from './chunk-NKS5ZIWN.js';
2
2
  import './chunk-MCIXU6XP.js';
@@ -1,5 +1,5 @@
1
- import { P as ProxyConfig } from './types-CgAEmIVE.js';
2
- import './types-Bv29uEAs.js';
1
+ import { P as ProxyConfig } from './types-su27kEw0.js';
2
+ import './types-BSSqsu_Z.js';
3
3
  import 'zod';
4
4
 
5
5
  /**
@@ -1,5 +1,5 @@
1
- import { createProxyHandler } from './chunk-GLIEEJMW.js';
2
- import './chunk-WXVBPMDN.js';
1
+ import { createProxyHandler } from './chunk-NDCZABOK.js';
2
+ import './chunk-NKS5ZIWN.js';
3
3
  import './chunk-MCIXU6XP.js';
4
4
 
5
5
  // src/tanstack-start.ts