realtime-avatar 0.27.5 → 0.29.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +48 -2
- package/dist/{chunk-IJHM44YT.js → chunk-HC7O44WE.js} +2 -1
- package/dist/{chunk-6E3SZH35.js → chunk-MMO5AQN4.js} +183 -65
- package/dist/express.d.ts +2 -2
- package/dist/express.js +2 -2
- package/dist/hono.d.ts +2 -2
- package/dist/hono.js +2 -2
- package/dist/index.d.ts +36 -6
- package/dist/index.js +1 -1
- package/dist/nextjs.d.ts +2 -2
- package/dist/nextjs.js +2 -2
- package/dist/{proxy-client-B7BAxMyL.d.ts → proxy-client-BJQxlVHz.d.ts} +67 -14
- package/dist/react-native.d.ts +2 -2
- package/dist/react-native.js +56 -7
- package/dist/react.d.ts +12 -3
- package/dist/react.js +60 -9
- package/dist/server.d.ts +2 -2
- package/dist/server.js +1 -1
- package/dist/tanstack-start.d.ts +2 -2
- package/dist/tanstack-start.js +2 -2
- package/dist/{types-C8X50QT9.d.ts → types-Bv29uEAs.d.ts} +278 -10
- package/dist/{types-B1IR4EaJ.d.ts → types-CgAEmIVE.d.ts} +1 -1
- package/package.json +1 -1
|
@@ -19,6 +19,34 @@ declare function useAvatarCamera({ allowed, active }: {
|
|
|
19
19
|
participant: livekit_client.LocalParticipant;
|
|
20
20
|
};
|
|
21
21
|
|
|
22
|
+
declare const voiceErrorFrameSchema: z.ZodObject<{
|
|
23
|
+
kind: z.ZodLiteral<"voice_error">;
|
|
24
|
+
code: z.ZodEnum<{
|
|
25
|
+
voice_webhook_url_not_allowed: "voice_webhook_url_not_allowed";
|
|
26
|
+
voice_webhook_unauthorized: "voice_webhook_unauthorized";
|
|
27
|
+
voice_webhook_unreachable: "voice_webhook_unreachable";
|
|
28
|
+
voice_webhook_timeout: "voice_webhook_timeout";
|
|
29
|
+
voice_webhook_http_error: "voice_webhook_http_error";
|
|
30
|
+
voice_webhook_unsupported_audio: "voice_webhook_unsupported_audio";
|
|
31
|
+
voice_webhook_empty_audio: "voice_webhook_empty_audio";
|
|
32
|
+
}>;
|
|
33
|
+
message: z.ZodString;
|
|
34
|
+
status: z.ZodOptional<z.ZodInt>;
|
|
35
|
+
fatal: z.ZodBoolean;
|
|
36
|
+
request_id: z.ZodOptional<z.ZodString>;
|
|
37
|
+
}, z.core.$strip>;
|
|
38
|
+
|
|
39
|
+
declare const fishTtsModelSchema: z.ZodUnion<readonly [z.ZodEnum<{
|
|
40
|
+
s1: "s1";
|
|
41
|
+
"s2-pro": "s2-pro";
|
|
42
|
+
"s2.1-pro": "s2.1-pro";
|
|
43
|
+
"drama-3": "drama-3";
|
|
44
|
+
}>, z.ZodEnum<{
|
|
45
|
+
"speech-1.6": "speech-1.6";
|
|
46
|
+
"speech-1.5": "speech-1.5";
|
|
47
|
+
"s1-mini": "s1-mini";
|
|
48
|
+
}>]>;
|
|
49
|
+
|
|
22
50
|
/**
|
|
23
51
|
* Wire schemas for the public Realtime Avatar API.
|
|
24
52
|
*
|
|
@@ -53,13 +81,6 @@ declare const cartesiaTtsModelSchema: z.ZodEnum<{
|
|
|
53
81
|
"cartesia/sonic-turbo": "cartesia/sonic-turbo";
|
|
54
82
|
"cartesia/sonic-turbo-latest": "cartesia/sonic-turbo-latest";
|
|
55
83
|
}>;
|
|
56
|
-
declare const fishTtsModelSchema: z.ZodEnum<{
|
|
57
|
-
"speech-1.6": "speech-1.6";
|
|
58
|
-
s1: "s1";
|
|
59
|
-
"s2-pro": "s2-pro";
|
|
60
|
-
"speech-1.5": "speech-1.5";
|
|
61
|
-
"s1-mini": "s1-mini";
|
|
62
|
-
}>;
|
|
63
84
|
declare const voiceSpecSchema: z.ZodDiscriminatedUnion<[z.ZodObject<{
|
|
64
85
|
provider: z.ZodLiteral<"cartesia">;
|
|
65
86
|
model: z.ZodDefault<z.ZodEnum<{
|
|
@@ -83,13 +104,16 @@ declare const voiceSpecSchema: z.ZodDiscriminatedUnion<[z.ZodObject<{
|
|
|
83
104
|
language: z.ZodOptional<z.ZodNullable<z.ZodString>>;
|
|
84
105
|
}, z.core.$strict>, z.ZodObject<{
|
|
85
106
|
provider: z.ZodLiteral<"fish">;
|
|
86
|
-
model: z.ZodDefault<z.ZodEnum<{
|
|
87
|
-
"speech-1.6": "speech-1.6";
|
|
107
|
+
model: z.ZodDefault<z.ZodUnion<readonly [z.ZodEnum<{
|
|
88
108
|
s1: "s1";
|
|
89
109
|
"s2-pro": "s2-pro";
|
|
110
|
+
"s2.1-pro": "s2.1-pro";
|
|
111
|
+
"drama-3": "drama-3";
|
|
112
|
+
}>, z.ZodEnum<{
|
|
113
|
+
"speech-1.6": "speech-1.6";
|
|
90
114
|
"speech-1.5": "speech-1.5";
|
|
91
115
|
"s1-mini": "s1-mini";
|
|
92
|
-
}>>;
|
|
116
|
+
}>]>>;
|
|
93
117
|
voice_id: z.ZodString;
|
|
94
118
|
speed: z.ZodOptional<z.ZodNullable<z.ZodNumber>>;
|
|
95
119
|
emotion: z.ZodOptional<z.ZodNullable<z.ZodString>>;
|
|
@@ -188,13 +212,16 @@ declare const liveKitSessionRequestSchema: z.ZodObject<{
|
|
|
188
212
|
language: z.ZodOptional<z.ZodNullable<z.ZodString>>;
|
|
189
213
|
}, z.core.$strict>, z.ZodObject<{
|
|
190
214
|
provider: z.ZodLiteral<"fish">;
|
|
191
|
-
model: z.ZodDefault<z.ZodEnum<{
|
|
192
|
-
"speech-1.6": "speech-1.6";
|
|
215
|
+
model: z.ZodDefault<z.ZodUnion<readonly [z.ZodEnum<{
|
|
193
216
|
s1: "s1";
|
|
194
217
|
"s2-pro": "s2-pro";
|
|
218
|
+
"s2.1-pro": "s2.1-pro";
|
|
219
|
+
"drama-3": "drama-3";
|
|
220
|
+
}>, z.ZodEnum<{
|
|
221
|
+
"speech-1.6": "speech-1.6";
|
|
195
222
|
"speech-1.5": "speech-1.5";
|
|
196
223
|
"s1-mini": "s1-mini";
|
|
197
|
-
}>>;
|
|
224
|
+
}>]>>;
|
|
198
225
|
voice_id: z.ZodString;
|
|
199
226
|
speed: z.ZodOptional<z.ZodNullable<z.ZodNumber>>;
|
|
200
227
|
emotion: z.ZodOptional<z.ZodNullable<z.ZodString>>;
|
|
@@ -482,6 +509,8 @@ declare const approachingEndReasonSchema: z.ZodEnum<{
|
|
|
482
509
|
type ApproachingEndReason = z.infer<typeof approachingEndReasonSchema>;
|
|
483
510
|
declare const knownBehaviorStates: readonly ["idle", "listening", "thinking", "speaking"];
|
|
484
511
|
type KnownBehaviorState = (typeof knownBehaviorStates)[number];
|
|
512
|
+
/** Why the caller's voice webhook failed. Codes this SDK predates arrive as plain strings. */
|
|
513
|
+
type VoiceErrorCode = z.infer<typeof voiceErrorFrameSchema>["code"] | (string & {});
|
|
485
514
|
declare const capacityBusyResponseSchema: z.ZodObject<{
|
|
486
515
|
message: z.ZodString;
|
|
487
516
|
capacity: z.ZodObject<{
|
|
@@ -1818,6 +1847,22 @@ type ClipResult = {
|
|
|
1818
1847
|
accepted: boolean;
|
|
1819
1848
|
reason: string;
|
|
1820
1849
|
};
|
|
1850
|
+
|
|
1851
|
+
/**
|
|
1852
|
+
* Your voice webhook failed (a call minted with `voiceWebhook`), so the rest of that reply is not
|
|
1853
|
+
* spoken: the sentences after the failed one are not POSTed, and the next reply tries again.
|
|
1854
|
+
* `message` is one developer-facing sentence, never your URL, secret or response body. `status` is
|
|
1855
|
+
* the HTTP status when your endpoint answered. `fatal` means the call is ending: a 401 or 403, a
|
|
1856
|
+
* URL the worker refuses to call, or repeated failures, which end it with the last failure's code,
|
|
1857
|
+
* whatever that code is. `onEnded` follows with reason `failed`.
|
|
1858
|
+
*/
|
|
1859
|
+
type VoiceError = {
|
|
1860
|
+
code: VoiceErrorCode;
|
|
1861
|
+
message: string;
|
|
1862
|
+
status?: number;
|
|
1863
|
+
fatal: boolean;
|
|
1864
|
+
requestId?: string;
|
|
1865
|
+
};
|
|
1821
1866
|
/**
|
|
1822
1867
|
* The `useRealtimeSession` input. ADDITIVE over {@link UseSessionLifecycleInput}
|
|
1823
1868
|
* (whose `client`/`session`/`active`/`idleSeconds`/`idleWarnLeadSeconds`/reconnect
|
|
@@ -1869,6 +1914,8 @@ type UseRealtimeSessionInput<T extends LLMProvider = LLMProvider> = UseSessionLi
|
|
|
1869
1914
|
onBehaviorChange?: (b: BehaviorSnapshot) => void;
|
|
1870
1915
|
/** A clip request was answered (also resolved on the performAction promise). */
|
|
1871
1916
|
onClipResult?: (r: ClipResult) => void;
|
|
1917
|
+
/** Your voice webhook failed for a sentence; see {@link VoiceError}. Fires once per frame the worker sends. */
|
|
1918
|
+
onVoiceError?: (error: VoiceError) => void;
|
|
1872
1919
|
};
|
|
1873
1920
|
/**
|
|
1874
1921
|
* The composed realtime-session surface. `phase` is the {@link SessionLifecyclePhase}
|
|
@@ -1947,6 +1994,12 @@ type RealtimeSessionApi = SessionLifecycleApi & {
|
|
|
1947
1994
|
end: (reason?: EndReason) => void;
|
|
1948
1995
|
/** The avatar's live nonverbal behavior, or null pre-choreo (see {@link BehaviorSnapshot}). */
|
|
1949
1996
|
behavior: BehaviorSnapshot | null;
|
|
1997
|
+
/**
|
|
1998
|
+
* The last {@link VoiceError} of this call, or null. It outlives the call it ended; the next call
|
|
1999
|
+
* starts at null, whether it starts by deactivating and reactivating, a new avatar or mode, a
|
|
2000
|
+
* redial, or `reset()`.
|
|
2001
|
+
*/
|
|
2002
|
+
voiceError: VoiceError | null;
|
|
1950
2003
|
/** Request a declared semantic action; acceptance schedules it and does not prove playback. */
|
|
1951
2004
|
performAction: (actionId: string, opts?: {
|
|
1952
2005
|
timeoutMs?: number;
|
|
@@ -2084,4 +2137,4 @@ interface ProxyClientOptions {
|
|
|
2084
2137
|
}
|
|
2085
2138
|
declare function createProxyClient(options: ProxyClientOptions): AvatarSessionClient;
|
|
2086
2139
|
|
|
2087
|
-
export { MAX_SESSION_INSTRUCTIONS_CHARS as $, type AvatarVideoFit as A, type BehaviorSnapshot as B, type CallTranscript as C, DEFAULT_APPROACHING_END_LEAD_SECONDS as D, type EndReason as E, type FishTtsModel as F, type Governor as G, type GovernorConfig as H, type GovernorSignal as I, type GovernorState as J, type GraceWindowClosedEvent as K, type LiveKitSessionGrant as L, type GraceWindowOpenEvent as M, type GraceWindowState as N, type IdleWarningEvent as O, type InboundRtpCursor as P, type InboundRtpReading as Q, type KnownBehaviorState as R, type LLMProvider as S, type LLMSelection as T, type LiveKitAvatarGrantState as U, type LiveKitAvatarGrantStatus as V, type LiveKitCapacityState as W, type LiveKitConnectionStatus as X, type LiveKitSessionRequest as Y, type LiveKitSessionStartResult as Z, MAX_RECONNECT_ATTEMPTS as _, AdaptivePlayoutController as a, type
|
|
2140
|
+
export { MAX_SESSION_INSTRUCTIONS_CHARS as $, type AvatarVideoFit as A, type BehaviorSnapshot as B, type CallTranscript as C, DEFAULT_APPROACHING_END_LEAD_SECONDS as D, type EndReason as E, type FishTtsModel as F, type Governor as G, type GovernorConfig as H, type GovernorSignal as I, type GovernorState as J, type GraceWindowClosedEvent as K, type LiveKitSessionGrant as L, type GraceWindowOpenEvent as M, type GraceWindowState as N, type IdleWarningEvent as O, type InboundRtpCursor as P, type InboundRtpReading as Q, type KnownBehaviorState as R, type LLMProvider as S, type LLMSelection as T, type LiveKitAvatarGrantState as U, type LiveKitAvatarGrantStatus as V, type LiveKitCapacityState as W, type LiveKitConnectionStatus as X, type LiveKitSessionRequest as Y, type LiveKitSessionStartResult as Z, MAX_RECONNECT_ATTEMPTS as _, AdaptivePlayoutController as a, type MicLease as a$, type ProxyClientOptions as a0, type QualityCap as a1, RECONNECT_BACKOFF_MS as a2, type RealtimeAvatarRequestOptions as a3, type RealtimeSessionApi as a4, type RealtimeSessionMedia as a5, type RealtimeSessionRoomSinks as a6, type ReconnectPolicy as a7, type ReconnectingEvent as a8, type RecoveryState as a9, capacityStateFromGrant as aA, createProxyClient as aB, isNativeLiveTrackSubscribed as aC, knownBehaviorStates as aD, mapTurnState as aE, readInboundRtp as aF, sessionBehaviorSchema as aG, sessionClipSchema as aH, splitCallTranscript as aI, useAvatarAdaptivePlayoutDelay as aJ, useAvatarCamera as aK, useAvatarPlayoutDelay as aL, useAvatarQualityGovernor as aM, useCallTranscript as aN, useLiveKitAvatarGrant as aO, useLiveTrackProducing as aP, useMicLease as aQ, useRealtimeSession as aR, useReleaseMicLeaseOnTrackEnded as aS, useSessionLifecycle as aT, type CallMicrophone as aU, type CallAudioPlayback as aV, type CallMicrophoneProblem as aW, AvatarVideoSurface as aX, type AvatarVideoSurfaceProps as aY, type LivePlaybackKeeper as aZ, MIC_LEASE_ENDED_TIMEOUT_MS as a_, type RetryStep as aa, type SendTurnOptions as ab, type SessionBehavior as ac, type SessionClip as ad, type SessionClocks as ae, type SessionEndReason as af, type SessionLifecycleApi as ag, type SessionLifecyclePhase as ah, type SessionLifecyclePhaseKind as ai, SessionLifecycleRoomBridge as aj, type SessionLifecycleRoomBridgeProps as ak, type SessionReadinessFailure as al, type SurfaceLayers as am, type TranscriptSender as an, type TranscriptSenderOptions as ao, type TurnState as ap, type TurnTimeoutEvent as aq, type UseAvatarQualityGovernorInput as ar, type UseLiveKitAvatarGrantInput as as, type UseRealtimeSessionInput as at, type UseSessionLifecycleInput as au, type VoiceError as av, type VoiceErrorCode as aw, type VoiceSpec as ax, type VoiceSpecInput as ay, capacityErrorFromBusy as az, type AdaptivePlayoutDecision as b, NETWORK_QUALITY_POLICY as b0, type NetworkQualityStatus as b1, type PlayableVideoElement as b2, RealtimeAvatarCapacityError as b3, RealtimeAvatarLiveKitRoom as b4, type RealtimeAvatarLiveKitRoomProps as b5, shouldReplayPendingTurn as b6, type AdaptivePlayoutOptions as c, type AdaptivePlayoutSample as d, type ApproachingEndEvent as e, type ApproachingEndReason as f, type AvatarConnectionDetails as g, type AvatarSessionClient as h, type CallTranscriptSegment as i, type CartesiaTtsModel as j, type ClipResult as k, type ClosingTurnResult as l, type CreditsLowEvent as m, DEFAULT_CREDITS_LOW_LEAD_SECONDS as n, DEFAULT_GOVERNOR_CONFIG as o, DEFAULT_GRACE_CEILING_SECONDS as p, DEFAULT_GRACE_WINDOW_LEAD_SECONDS as q, DEFAULT_IDLE_SECONDS as r, DEFAULT_IDLE_WARN_LEAD_SECONDS as s, DEFAULT_READY_TIMEOUT_SECONDS as t, DEFAULT_TURN_TIMEOUT_SECONDS as u, type DeclaredInputSource as v, type EndedEvent as w, type ExtendResult as x, type FreezeReadingFn as y, type GovernorAction as z };
|
package/dist/react-native.d.ts
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
import { LiveKitRoomProps } from '@livekit/react-native';
|
|
2
2
|
export { AudioSession, VideoTrack, VideoTrackProps, registerGlobals } from '@livekit/react-native';
|
|
3
|
-
import { L as LiveKitSessionGrant, A as AvatarVideoFit } from './proxy-client-
|
|
4
|
-
export { a as AdaptivePlayoutController, b as AdaptivePlayoutDecision, c as AdaptivePlayoutOptions, d as AdaptivePlayoutSample, e as ApproachingEndEvent, f as ApproachingEndReason, g as AvatarConnectionDetails, h as AvatarSessionClient, B as BehaviorSnapshot, C as CallTranscript, i as CallTranscriptSegment, j as CartesiaTtsModel, k as ClipResult, l as ClosingTurnResult, m as CreditsLowEvent, D as DEFAULT_APPROACHING_END_LEAD_SECONDS, n as DEFAULT_CREDITS_LOW_LEAD_SECONDS, o as DEFAULT_GOVERNOR_CONFIG, p as DEFAULT_GRACE_CEILING_SECONDS, q as DEFAULT_GRACE_WINDOW_LEAD_SECONDS, r as DEFAULT_IDLE_SECONDS, s as DEFAULT_IDLE_WARN_LEAD_SECONDS, t as DEFAULT_READY_TIMEOUT_SECONDS, u as DEFAULT_TURN_TIMEOUT_SECONDS, v as DeclaredInputSource, E as EndReason, w as EndedEvent, x as ExtendResult, F as FishTtsModel, y as FreezeReadingFn, G as Governor, z as GovernorAction, H as GovernorConfig, I as GovernorSignal, J as GovernorState, K as GraceWindowClosedEvent, M as GraceWindowOpenEvent, N as GraceWindowState, O as IdleWarningEvent, P as InboundRtpCursor, Q as InboundRtpReading, R as KnownBehaviorState, S as LLMProvider, T as LLMSelection, U as LiveKitAvatarGrantState, V as LiveKitAvatarGrantStatus, W as LiveKitCapacityState, X as LiveKitConnectionStatus, Y as LiveKitSessionRequest, Z as LiveKitSessionStartResult, _ as MAX_RECONNECT_ATTEMPTS, $ as MAX_SESSION_INSTRUCTIONS_CHARS, a0 as ProxyClientOptions, a1 as QualityCap, a2 as RECONNECT_BACKOFF_MS, a3 as RealtimeAvatarRequestOptions, a4 as RealtimeSessionApi, a5 as RealtimeSessionMedia, a6 as RealtimeSessionRoomSinks, a7 as ReconnectPolicy, a8 as ReconnectingEvent, a9 as RecoveryState, aa as RetryStep, ab as SendTurnOptions, ac as SessionBehavior, ad as SessionClip, ae as SessionClocks, af as SessionEndReason, ag as SessionLifecycleApi, ah as SessionLifecyclePhase, ai as SessionLifecyclePhaseKind, aj as SessionLifecycleRoomBridge, ak as SessionLifecycleRoomBridgeProps, al as SessionReadinessFailure, am as SurfaceLayers, an as TranscriptSender, ao as TranscriptSenderOptions, ap as TurnState, aq as TurnTimeoutEvent, ar as UseAvatarQualityGovernorInput, as as UseLiveKitAvatarGrantInput, at as UseRealtimeSessionInput, au as UseSessionLifecycleInput, av as
|
|
3
|
+
import { L as LiveKitSessionGrant, A as AvatarVideoFit } from './proxy-client-BJQxlVHz.js';
|
|
4
|
+
export { a as AdaptivePlayoutController, b as AdaptivePlayoutDecision, c as AdaptivePlayoutOptions, d as AdaptivePlayoutSample, e as ApproachingEndEvent, f as ApproachingEndReason, g as AvatarConnectionDetails, h as AvatarSessionClient, B as BehaviorSnapshot, C as CallTranscript, i as CallTranscriptSegment, j as CartesiaTtsModel, k as ClipResult, l as ClosingTurnResult, m as CreditsLowEvent, D as DEFAULT_APPROACHING_END_LEAD_SECONDS, n as DEFAULT_CREDITS_LOW_LEAD_SECONDS, o as DEFAULT_GOVERNOR_CONFIG, p as DEFAULT_GRACE_CEILING_SECONDS, q as DEFAULT_GRACE_WINDOW_LEAD_SECONDS, r as DEFAULT_IDLE_SECONDS, s as DEFAULT_IDLE_WARN_LEAD_SECONDS, t as DEFAULT_READY_TIMEOUT_SECONDS, u as DEFAULT_TURN_TIMEOUT_SECONDS, v as DeclaredInputSource, E as EndReason, w as EndedEvent, x as ExtendResult, F as FishTtsModel, y as FreezeReadingFn, G as Governor, z as GovernorAction, H as GovernorConfig, I as GovernorSignal, J as GovernorState, K as GraceWindowClosedEvent, M as GraceWindowOpenEvent, N as GraceWindowState, O as IdleWarningEvent, P as InboundRtpCursor, Q as InboundRtpReading, R as KnownBehaviorState, S as LLMProvider, T as LLMSelection, U as LiveKitAvatarGrantState, V as LiveKitAvatarGrantStatus, W as LiveKitCapacityState, X as LiveKitConnectionStatus, Y as LiveKitSessionRequest, Z as LiveKitSessionStartResult, _ as MAX_RECONNECT_ATTEMPTS, $ as MAX_SESSION_INSTRUCTIONS_CHARS, a0 as ProxyClientOptions, a1 as QualityCap, a2 as RECONNECT_BACKOFF_MS, a3 as RealtimeAvatarRequestOptions, a4 as RealtimeSessionApi, a5 as RealtimeSessionMedia, a6 as RealtimeSessionRoomSinks, a7 as ReconnectPolicy, a8 as ReconnectingEvent, a9 as RecoveryState, aa as RetryStep, ab as SendTurnOptions, ac as SessionBehavior, ad as SessionClip, ae as SessionClocks, af as SessionEndReason, ag as SessionLifecycleApi, ah as SessionLifecyclePhase, ai as SessionLifecyclePhaseKind, aj as SessionLifecycleRoomBridge, ak as SessionLifecycleRoomBridgeProps, al as SessionReadinessFailure, am as SurfaceLayers, an as TranscriptSender, ao as TranscriptSenderOptions, ap as TurnState, aq as TurnTimeoutEvent, ar as UseAvatarQualityGovernorInput, as as UseLiveKitAvatarGrantInput, at as UseRealtimeSessionInput, au as UseSessionLifecycleInput, av as VoiceError, aw as VoiceErrorCode, ax as VoiceSpec, ay as VoiceSpecInput, az as capacityErrorFromBusy, aA as capacityStateFromGrant, aB as createProxyClient, aC as isNativeLiveTrackSubscribed, aD as knownBehaviorStates, aE as mapTurnState, aF as readInboundRtp, aG as sessionBehaviorSchema, aH as sessionClipSchema, aI as splitCallTranscript, aJ as useAvatarAdaptivePlayoutDelay, aK as useAvatarCamera, aL as useAvatarPlayoutDelay, aM as useAvatarQualityGovernor, aN as useCallTranscript, aO as useLiveKitAvatarGrant, aP as useLiveTrackProducing, aQ as useMicLease, aR as useRealtimeSession, aS as useReleaseMicLeaseOnTrackEnded, aT as useSessionLifecycle } from './proxy-client-BJQxlVHz.js';
|
|
5
5
|
import { ReactNode, ReactElement } from 'react';
|
|
6
6
|
import { StyleProp, ViewStyle } from 'react-native';
|
|
7
7
|
export { useChat, useConnectionState, useLocalParticipant, useRoomContext, useTrackToggle, useTranscriptions, useVoiceAssistant } from '@livekit/components-react';
|
package/dist/react-native.js
CHANGED
|
@@ -324,6 +324,7 @@ var ERROR_SEMANTICS = {
|
|
|
324
324
|
"service_warming": { statuses: [503], message: "Realtime Avatar is warming up. Try again in a moment.", retryable: true },
|
|
325
325
|
"avatar_not_ready": { statuses: [409], message: "This avatar is still being built \u2014 it is usually ready within a minute. Try again then.", retryable: true },
|
|
326
326
|
"clip_library_not_ready": { statuses: [409], message: "This avatar's motion clips are still rendering. Try again in a minute.", retryable: true },
|
|
327
|
+
"avatar_not_callable": { statuses: [409], message: "This avatar only hosts a live stream and cannot take calls. Choose another avatar.", retryable: false },
|
|
327
328
|
"recording_unsupported": { statuses: [501], message: "Session recording is not enabled for this deployment.", retryable: false },
|
|
328
329
|
"recording_requires_new_room": { statuses: [422], message: "Recording requires a new session room. Start a fresh session to record it.", retryable: false },
|
|
329
330
|
"recording_unavailable": { statuses: [503], message: "Session recording is temporarily unavailable. Try again in a moment.", retryable: true }
|
|
@@ -1725,6 +1726,25 @@ var transcriptionOptionsSchema = z.object({
|
|
|
1725
1726
|
}).strict();
|
|
1726
1727
|
var schema02 = z.enum(["minor-sexual-content-v1", "safe-content-v1"]);
|
|
1727
1728
|
var contentPolicySchema = schema02;
|
|
1729
|
+
var schema03 = z.literal("voice_error");
|
|
1730
|
+
var schema12 = z.enum(["voice_webhook_url_not_allowed", "voice_webhook_unauthorized", "voice_webhook_unreachable", "voice_webhook_timeout", "voice_webhook_http_error", "voice_webhook_unsupported_audio", "voice_webhook_empty_audio"]);
|
|
1731
|
+
var schema22 = z.string();
|
|
1732
|
+
var schema32 = z.int().min(-9007199254740991).max(9007199254740991);
|
|
1733
|
+
var schema42 = z.boolean();
|
|
1734
|
+
var schema5 = z.object({
|
|
1735
|
+
"kind": schema03,
|
|
1736
|
+
"code": schema12,
|
|
1737
|
+
"message": schema22,
|
|
1738
|
+
"status": schema32.optional(),
|
|
1739
|
+
"fatal": schema42,
|
|
1740
|
+
"request_id": schema22.optional()
|
|
1741
|
+
});
|
|
1742
|
+
var voiceErrorFrameSchema = schema5;
|
|
1743
|
+
var schema04 = z.enum(["s1", "s2-pro", "s2.1-pro", "drama-3"]);
|
|
1744
|
+
var schema13 = z.enum(["speech-1.6", "speech-1.5", "s1-mini"]);
|
|
1745
|
+
var schema23 = z.union([schema04, schema13]);
|
|
1746
|
+
var fishTtsModelSchema = schema23;
|
|
1747
|
+
var DEFAULT_FISH_TTS_MODEL = "s2.1-pro";
|
|
1728
1748
|
|
|
1729
1749
|
// ../client/src/wire.ts
|
|
1730
1750
|
var DEFAULT_AVATAR_ID = "maria";
|
|
@@ -1779,8 +1799,6 @@ var cartesiaVoiceSpecSchema = z.object({
|
|
|
1779
1799
|
emotion: z.string().min(1).max(80).nullable().optional(),
|
|
1780
1800
|
language: z.string().min(2).max(16).nullable().optional()
|
|
1781
1801
|
}).strict();
|
|
1782
|
-
var FISH_TTS_MODELS = ["speech-1.6", "s1", "s2-pro", "speech-1.5", "s1-mini"];
|
|
1783
|
-
var fishTtsModelSchema = z.enum(FISH_TTS_MODELS);
|
|
1784
1802
|
var breezeVoiceSpecSchema = z.object({
|
|
1785
1803
|
provider: z.literal("breezeblue"),
|
|
1786
1804
|
model: z.string().min(1).max(80).default("bluebell-v1-en"),
|
|
@@ -1791,7 +1809,7 @@ var breezeVoiceSpecSchema = z.object({
|
|
|
1791
1809
|
}).strict();
|
|
1792
1810
|
var fishVoiceSpecSchema = z.object({
|
|
1793
1811
|
provider: z.literal("fish"),
|
|
1794
|
-
model: fishTtsModelSchema.default(
|
|
1812
|
+
model: fishTtsModelSchema.default(DEFAULT_FISH_TTS_MODEL),
|
|
1795
1813
|
voice_id: z.string().min(1).max(120),
|
|
1796
1814
|
speed: z.number().min(0.5).max(2).nullable().optional(),
|
|
1797
1815
|
emotion: z.string().min(1).max(80).nullable().optional(),
|
|
@@ -2026,13 +2044,15 @@ var clipAckFrameSchema = z.object({
|
|
|
2026
2044
|
accepted: z.boolean(),
|
|
2027
2045
|
reason: z.string().max(64)
|
|
2028
2046
|
}).strip();
|
|
2047
|
+
var voiceErrorFrameSchema2 = voiceErrorFrameSchema.extend({ code: z.string().min(1) });
|
|
2029
2048
|
var lifecycleServerFrameSchema = z.discriminatedUnion("kind", [
|
|
2030
2049
|
sessionClockFrameSchema,
|
|
2031
2050
|
endingFrameSchema,
|
|
2032
2051
|
closingTurnDoneFrameSchema,
|
|
2033
2052
|
endedFrameSchema,
|
|
2034
2053
|
behaviorStateFrameSchema,
|
|
2035
|
-
clipAckFrameSchema
|
|
2054
|
+
clipAckFrameSchema,
|
|
2055
|
+
voiceErrorFrameSchema2
|
|
2036
2056
|
]);
|
|
2037
2057
|
var liveKitCapacitySnapshotSchema = z.object({
|
|
2038
2058
|
// Placement identity + per-worker session ceiling. These are always present on the
|
|
@@ -3294,6 +3314,7 @@ function useRealtimeSession(input) {
|
|
|
3294
3314
|
onEnded,
|
|
3295
3315
|
onBehaviorChange,
|
|
3296
3316
|
onClipResult,
|
|
3317
|
+
onVoiceError,
|
|
3297
3318
|
// everything else (client/active/idleSeconds/idleWarnLeadSeconds/reconnect/requestOptions…)
|
|
3298
3319
|
// flows straight into the inner SSOT hook.
|
|
3299
3320
|
...inner
|
|
@@ -3309,6 +3330,7 @@ function useRealtimeSession(input) {
|
|
|
3309
3330
|
const [turn, setTurn] = useState("quiet");
|
|
3310
3331
|
const [media, setMediaState] = useState({ video: "connecting", audio: "silent" });
|
|
3311
3332
|
const [behavior, setBehavior] = useState(null);
|
|
3333
|
+
const [voiceError, setVoiceError] = useState(null);
|
|
3312
3334
|
const [microphoneFacts2, setMicrophoneFactsState] = useState({
|
|
3313
3335
|
wanted: false,
|
|
3314
3336
|
connected: false,
|
|
@@ -3322,8 +3344,8 @@ function useRealtimeSession(input) {
|
|
|
3322
3344
|
const lastBehaviorRef = useRef(null);
|
|
3323
3345
|
const lastLabeledEndReasonRef = useRef(null);
|
|
3324
3346
|
const pendingClipsRef = useRef(/* @__PURE__ */ new Map());
|
|
3325
|
-
const cbRef = useRef({ onApproachingEnd, onGraceWindowOpen, onGraceWindowClosed, onIdleWarning, onCreditsLow, onTurnTimeout, onReconnecting, onReconnected, onEnded, onBehaviorChange, onClipResult });
|
|
3326
|
-
cbRef.current = { onApproachingEnd, onGraceWindowOpen, onGraceWindowClosed, onIdleWarning, onCreditsLow, onTurnTimeout, onReconnecting, onReconnected, onEnded, onBehaviorChange, onClipResult };
|
|
3347
|
+
const cbRef = useRef({ onApproachingEnd, onGraceWindowOpen, onGraceWindowClosed, onIdleWarning, onCreditsLow, onTurnTimeout, onReconnecting, onReconnected, onEnded, onBehaviorChange, onClipResult, onVoiceError });
|
|
3348
|
+
cbRef.current = { onApproachingEnd, onGraceWindowOpen, onGraceWindowClosed, onIdleWarning, onCreditsLow, onTurnTimeout, onReconnecting, onReconnected, onEnded, onBehaviorChange, onClipResult, onVoiceError };
|
|
3327
3349
|
const approachingEndLeadMs = positive(approachingEndLeadSeconds, DEFAULT_APPROACHING_END_LEAD_SECONDS) * 1e3;
|
|
3328
3350
|
const graceWindowLeadMs = positive(graceWindowLeadSeconds, DEFAULT_GRACE_WINDOW_LEAD_SECONDS) * 1e3;
|
|
3329
3351
|
const graceCeilingMs = positive(graceCeilingSeconds, DEFAULT_GRACE_CEILING_SECONDS) * 1e3;
|
|
@@ -3411,6 +3433,18 @@ function useRealtimeSession(input) {
|
|
|
3411
3433
|
cbRef.current.onClipResult?.(result);
|
|
3412
3434
|
break;
|
|
3413
3435
|
}
|
|
3436
|
+
case "voice_error": {
|
|
3437
|
+
const error = {
|
|
3438
|
+
code: f.code,
|
|
3439
|
+
message: f.message,
|
|
3440
|
+
...f.status === void 0 ? {} : { status: f.status },
|
|
3441
|
+
fatal: f.fatal,
|
|
3442
|
+
...f.request_id === void 0 ? {} : { requestId: f.request_id }
|
|
3443
|
+
};
|
|
3444
|
+
setVoiceError(error);
|
|
3445
|
+
cbRef.current.onVoiceError?.(error);
|
|
3446
|
+
break;
|
|
3447
|
+
}
|
|
3414
3448
|
}
|
|
3415
3449
|
}, []);
|
|
3416
3450
|
const registerDataPublisher = useCallback((publish) => {
|
|
@@ -3605,10 +3639,22 @@ function useRealtimeSession(input) {
|
|
|
3605
3639
|
if (active) return;
|
|
3606
3640
|
lastLabeledEndReasonRef.current = null;
|
|
3607
3641
|
setMicrophoneMuted(false);
|
|
3642
|
+
setVoiceError(null);
|
|
3608
3643
|
}, [active]);
|
|
3644
|
+
const grantSessionId = lifecycle.grant?.session_id ?? null;
|
|
3645
|
+
const voiceErrorGrantRef = useRef(null);
|
|
3646
|
+
useEffect(() => {
|
|
3647
|
+
if (phaseKind === "requesting") setVoiceError(null);
|
|
3648
|
+
if (grantSessionId === null) return;
|
|
3649
|
+
if (voiceErrorGrantRef.current !== null && voiceErrorGrantRef.current !== grantSessionId) setVoiceError(null);
|
|
3650
|
+
voiceErrorGrantRef.current = grantSessionId;
|
|
3651
|
+
}, [phaseKind, grantSessionId]);
|
|
3609
3652
|
const innerReconnect = lifecycle.reconnect;
|
|
3610
3653
|
const reconnect = useCallback(() => {
|
|
3611
|
-
if (phaseKind === "ended")
|
|
3654
|
+
if (phaseKind === "ended") {
|
|
3655
|
+
lastLabeledEndReasonRef.current = null;
|
|
3656
|
+
setVoiceError(null);
|
|
3657
|
+
}
|
|
3612
3658
|
innerReconnect();
|
|
3613
3659
|
}, [innerReconnect, phaseKind]);
|
|
3614
3660
|
const performAction = useCallback(
|
|
@@ -3657,6 +3703,7 @@ function useRealtimeSession(input) {
|
|
|
3657
3703
|
creditsLowFiredRef.current = false;
|
|
3658
3704
|
lastTurnRef.current = null;
|
|
3659
3705
|
setMicrophoneMuted(false);
|
|
3706
|
+
setVoiceError(null);
|
|
3660
3707
|
lifecycle.reset();
|
|
3661
3708
|
}, [lifecycle]);
|
|
3662
3709
|
return useMemo(
|
|
@@ -3685,6 +3732,7 @@ function useRealtimeSession(input) {
|
|
|
3685
3732
|
retryTurn,
|
|
3686
3733
|
end,
|
|
3687
3734
|
behavior,
|
|
3735
|
+
voiceError,
|
|
3688
3736
|
performAction,
|
|
3689
3737
|
onLifecycleData,
|
|
3690
3738
|
registerDataPublisher,
|
|
@@ -3717,6 +3765,7 @@ function useRealtimeSession(input) {
|
|
|
3717
3765
|
retryTurn,
|
|
3718
3766
|
end,
|
|
3719
3767
|
behavior,
|
|
3768
|
+
voiceError,
|
|
3720
3769
|
performAction,
|
|
3721
3770
|
onLifecycleData,
|
|
3722
3771
|
registerDataPublisher,
|
package/dist/react.d.ts
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
import { ToolRegistration, RegisteredTool } from './tools.js';
|
|
2
|
-
import { ak as SessionLifecycleRoomBridgeProps, h as AvatarSessionClient, A as AvatarVideoFit,
|
|
3
|
-
export { a as AdaptivePlayoutController, b as AdaptivePlayoutDecision, c as AdaptivePlayoutOptions, d as AdaptivePlayoutSample, e as ApproachingEndEvent, f as ApproachingEndReason, g as AvatarConnectionDetails,
|
|
2
|
+
import { ak as SessionLifecycleRoomBridgeProps, h as AvatarSessionClient, A as AvatarVideoFit, aU as CallMicrophone, aV as CallAudioPlayback, av as VoiceError, aW as CallMicrophoneProblem } from './proxy-client-BJQxlVHz.js';
|
|
3
|
+
export { a as AdaptivePlayoutController, b as AdaptivePlayoutDecision, c as AdaptivePlayoutOptions, d as AdaptivePlayoutSample, e as ApproachingEndEvent, f as ApproachingEndReason, g as AvatarConnectionDetails, aX as AvatarVideoSurface, aY as AvatarVideoSurfaceProps, B as BehaviorSnapshot, C as CallTranscript, i as CallTranscriptSegment, j as CartesiaTtsModel, k as ClipResult, l as ClosingTurnResult, m as CreditsLowEvent, D as DEFAULT_APPROACHING_END_LEAD_SECONDS, n as DEFAULT_CREDITS_LOW_LEAD_SECONDS, o as DEFAULT_GOVERNOR_CONFIG, p as DEFAULT_GRACE_CEILING_SECONDS, q as DEFAULT_GRACE_WINDOW_LEAD_SECONDS, r as DEFAULT_IDLE_SECONDS, s as DEFAULT_IDLE_WARN_LEAD_SECONDS, t as DEFAULT_READY_TIMEOUT_SECONDS, u as DEFAULT_TURN_TIMEOUT_SECONDS, v as DeclaredInputSource, E as EndReason, w as EndedEvent, x as ExtendResult, F as FishTtsModel, y as FreezeReadingFn, G as Governor, z as GovernorAction, H as GovernorConfig, I as GovernorSignal, J as GovernorState, K as GraceWindowClosedEvent, M as GraceWindowOpenEvent, N as GraceWindowState, O as IdleWarningEvent, P as InboundRtpCursor, Q as InboundRtpReading, R as KnownBehaviorState, S as LLMProvider, T as LLMSelection, U as LiveKitAvatarGrantState, V as LiveKitAvatarGrantStatus, W as LiveKitCapacityState, X as LiveKitConnectionStatus, Y as LiveKitSessionRequest, Z as LiveKitSessionStartResult, aZ as LivePlaybackKeeper, _ as MAX_RECONNECT_ATTEMPTS, $ as MAX_SESSION_INSTRUCTIONS_CHARS, a_ as MIC_LEASE_ENDED_TIMEOUT_MS, a$ as MicLease, b0 as NETWORK_QUALITY_POLICY, b1 as NetworkQualityStatus, b2 as PlayableVideoElement, a0 as ProxyClientOptions, a1 as QualityCap, a2 as RECONNECT_BACKOFF_MS, b3 as RealtimeAvatarCapacityError, b4 as RealtimeAvatarLiveKitRoom, b5 as RealtimeAvatarLiveKitRoomProps, a3 as RealtimeAvatarRequestOptions, a4 as RealtimeSessionApi, a5 as RealtimeSessionMedia, a6 as RealtimeSessionRoomSinks, a7 as ReconnectPolicy, a8 as ReconnectingEvent, a9 as RecoveryState, aa as RetryStep, ab as SendTurnOptions, ac as SessionBehavior, ad as SessionClip, ae as SessionClocks, af as SessionEndReason, ag as SessionLifecycleApi, ah as SessionLifecyclePhase, ai as SessionLifecyclePhaseKind, aj as SessionLifecycleRoomBridge, al as SessionReadinessFailure, am as SurfaceLayers, an as TranscriptSender, ao as TranscriptSenderOptions, ap as TurnState, aq as TurnTimeoutEvent, ar as UseAvatarQualityGovernorInput, as as UseLiveKitAvatarGrantInput, at as UseRealtimeSessionInput, au as UseSessionLifecycleInput, aw as VoiceErrorCode, ax as VoiceSpec, ay as VoiceSpecInput, az as capacityErrorFromBusy, aA as capacityStateFromGrant, aB as createProxyClient, aD as knownBehaviorStates, aE as mapTurnState, aF as readInboundRtp, aG as sessionBehaviorSchema, aH as sessionClipSchema, b6 as shouldReplayPendingTurn, aI as splitCallTranscript, aJ as useAvatarAdaptivePlayoutDelay, aK as useAvatarCamera, aL as useAvatarPlayoutDelay, aM as useAvatarQualityGovernor, aN as useCallTranscript, aO as useLiveKitAvatarGrant, aQ as useMicLease, aR as useRealtimeSession, aS as useReleaseMicLeaseOnTrackEnded, aT as useSessionLifecycle } from './proxy-client-BJQxlVHz.js';
|
|
4
4
|
import { CSSProperties, ReactNode } from 'react';
|
|
5
5
|
import * as react_jsx_runtime from 'react/jsx-runtime';
|
|
6
6
|
import { R as RecordingArtifact } from './recording-ydFVySSd.js';
|
|
@@ -120,6 +120,8 @@ type AvatarCallHandle = {
|
|
|
120
120
|
keepAlive: () => void;
|
|
121
121
|
/** End now. */
|
|
122
122
|
end: () => void;
|
|
123
|
+
/** The last voice webhook failure of this call, or null. Only a call minted with `voiceWebhook` has one. */
|
|
124
|
+
voiceError: VoiceError | null;
|
|
123
125
|
};
|
|
124
126
|
type AvatarCallProps = Pick<SessionLifecycleRoomBridgeProps, "onConnectionDetailsChange"> & {
|
|
125
127
|
client: AvatarSessionClient;
|
|
@@ -171,6 +173,13 @@ type AvatarCallProps = Pick<SessionLifecycleRoomBridgeProps, "onConnectionDetail
|
|
|
171
173
|
* is live. Fires once per distinct problem. Show `hint`, then offer `retryMicrophone()`.
|
|
172
174
|
*/
|
|
173
175
|
onMicrophoneProblem?: (problem: AvatarCallMicrophoneProblem) => void;
|
|
176
|
+
/**
|
|
177
|
+
* Your voice webhook failed, so the rest of that reply is not spoken; the next reply tries again.
|
|
178
|
+
* Fires once per error the worker reports (it rate-limits them). When `fatal` (a 401 or 403, a
|
|
179
|
+
* refused URL, or repeated failures, carrying the last failure's code), the call is ending and
|
|
180
|
+
* `onEnded` follows with `failed`.
|
|
181
|
+
*/
|
|
182
|
+
onVoiceError?: (error: VoiceError) => void;
|
|
174
183
|
/**
|
|
175
184
|
* What to show while `audio` is `blocked` (the browser is muting her until a gesture):
|
|
176
185
|
* - `true` (default): a "Tap to turn on sound" button, top-centre over the video.
|
|
@@ -207,4 +216,4 @@ declare function RecordingPlayer({ assets, className }: {
|
|
|
207
216
|
className?: string;
|
|
208
217
|
}): react_jsx_runtime.JSX.Element;
|
|
209
218
|
|
|
210
|
-
export { AvatarCall, type AvatarCallAudio, type AvatarCallEndReason, type AvatarCallHandle, type AvatarCallMicrophone, type AvatarCallMicrophoneProblem, type AvatarCallProps, type AvatarCallStatus, AvatarSessionClient, AvatarVideoFit, type CharacterToolsState, DEFAULT_AVATAR_UNSTABLE_STALL_MS, type RecordingPlaybackAsset, RecordingPlayer, SessionLifecycleRoomBridgeProps, useAvatarCall, useCharacterTools };
|
|
219
|
+
export { AvatarCall, type AvatarCallAudio, type AvatarCallEndReason, type AvatarCallHandle, type AvatarCallMicrophone, type AvatarCallMicrophoneProblem, type AvatarCallProps, type AvatarCallStatus, AvatarSessionClient, AvatarVideoFit, type CharacterToolsState, DEFAULT_AVATAR_UNSTABLE_STALL_MS, type RecordingPlaybackAsset, RecordingPlayer, SessionLifecycleRoomBridgeProps, VoiceError, useAvatarCall, useCharacterTools };
|
package/dist/react.js
CHANGED
|
@@ -530,6 +530,7 @@ var ERROR_SEMANTICS = {
|
|
|
530
530
|
"service_warming": { statuses: [503], message: "Realtime Avatar is warming up. Try again in a moment.", retryable: true },
|
|
531
531
|
"avatar_not_ready": { statuses: [409], message: "This avatar is still being built \u2014 it is usually ready within a minute. Try again then.", retryable: true },
|
|
532
532
|
"clip_library_not_ready": { statuses: [409], message: "This avatar's motion clips are still rendering. Try again in a minute.", retryable: true },
|
|
533
|
+
"avatar_not_callable": { statuses: [409], message: "This avatar only hosts a live stream and cannot take calls. Choose another avatar.", retryable: false },
|
|
533
534
|
"recording_unsupported": { statuses: [501], message: "Session recording is not enabled for this deployment.", retryable: false },
|
|
534
535
|
"recording_requires_new_room": { statuses: [422], message: "Recording requires a new session room. Start a fresh session to record it.", retryable: false },
|
|
535
536
|
"recording_unavailable": { statuses: [503], message: "Session recording is temporarily unavailable. Try again in a moment.", retryable: true }
|
|
@@ -2344,6 +2345,25 @@ var transcriptionOptionsSchema = z.object({
|
|
|
2344
2345
|
}).strict();
|
|
2345
2346
|
var schema02 = z.enum(["minor-sexual-content-v1", "safe-content-v1"]);
|
|
2346
2347
|
var contentPolicySchema = schema02;
|
|
2348
|
+
var schema03 = z.literal("voice_error");
|
|
2349
|
+
var schema12 = z.enum(["voice_webhook_url_not_allowed", "voice_webhook_unauthorized", "voice_webhook_unreachable", "voice_webhook_timeout", "voice_webhook_http_error", "voice_webhook_unsupported_audio", "voice_webhook_empty_audio"]);
|
|
2350
|
+
var schema22 = z.string();
|
|
2351
|
+
var schema32 = z.int().min(-9007199254740991).max(9007199254740991);
|
|
2352
|
+
var schema42 = z.boolean();
|
|
2353
|
+
var schema5 = z.object({
|
|
2354
|
+
"kind": schema03,
|
|
2355
|
+
"code": schema12,
|
|
2356
|
+
"message": schema22,
|
|
2357
|
+
"status": schema32.optional(),
|
|
2358
|
+
"fatal": schema42,
|
|
2359
|
+
"request_id": schema22.optional()
|
|
2360
|
+
});
|
|
2361
|
+
var voiceErrorFrameSchema = schema5;
|
|
2362
|
+
var schema04 = z.enum(["s1", "s2-pro", "s2.1-pro", "drama-3"]);
|
|
2363
|
+
var schema13 = z.enum(["speech-1.6", "speech-1.5", "s1-mini"]);
|
|
2364
|
+
var schema23 = z.union([schema04, schema13]);
|
|
2365
|
+
var fishTtsModelSchema = schema23;
|
|
2366
|
+
var DEFAULT_FISH_TTS_MODEL = "s2.1-pro";
|
|
2347
2367
|
|
|
2348
2368
|
// ../client/src/wire.ts
|
|
2349
2369
|
var DEFAULT_AVATAR_ID = "maria";
|
|
@@ -2398,8 +2418,6 @@ var cartesiaVoiceSpecSchema = z.object({
|
|
|
2398
2418
|
emotion: z.string().min(1).max(80).nullable().optional(),
|
|
2399
2419
|
language: z.string().min(2).max(16).nullable().optional()
|
|
2400
2420
|
}).strict();
|
|
2401
|
-
var FISH_TTS_MODELS = ["speech-1.6", "s1", "s2-pro", "speech-1.5", "s1-mini"];
|
|
2402
|
-
var fishTtsModelSchema = z.enum(FISH_TTS_MODELS);
|
|
2403
2421
|
var breezeVoiceSpecSchema = z.object({
|
|
2404
2422
|
provider: z.literal("breezeblue"),
|
|
2405
2423
|
model: z.string().min(1).max(80).default("bluebell-v1-en"),
|
|
@@ -2410,7 +2428,7 @@ var breezeVoiceSpecSchema = z.object({
|
|
|
2410
2428
|
}).strict();
|
|
2411
2429
|
var fishVoiceSpecSchema = z.object({
|
|
2412
2430
|
provider: z.literal("fish"),
|
|
2413
|
-
model: fishTtsModelSchema.default(
|
|
2431
|
+
model: fishTtsModelSchema.default(DEFAULT_FISH_TTS_MODEL),
|
|
2414
2432
|
voice_id: z.string().min(1).max(120),
|
|
2415
2433
|
speed: z.number().min(0.5).max(2).nullable().optional(),
|
|
2416
2434
|
emotion: z.string().min(1).max(80).nullable().optional(),
|
|
@@ -2645,13 +2663,15 @@ var clipAckFrameSchema = z.object({
|
|
|
2645
2663
|
accepted: z.boolean(),
|
|
2646
2664
|
reason: z.string().max(64)
|
|
2647
2665
|
}).strip();
|
|
2666
|
+
var voiceErrorFrameSchema2 = voiceErrorFrameSchema.extend({ code: z.string().min(1) });
|
|
2648
2667
|
var lifecycleServerFrameSchema = z.discriminatedUnion("kind", [
|
|
2649
2668
|
sessionClockFrameSchema,
|
|
2650
2669
|
endingFrameSchema,
|
|
2651
2670
|
closingTurnDoneFrameSchema,
|
|
2652
2671
|
endedFrameSchema,
|
|
2653
2672
|
behaviorStateFrameSchema,
|
|
2654
|
-
clipAckFrameSchema
|
|
2673
|
+
clipAckFrameSchema,
|
|
2674
|
+
voiceErrorFrameSchema2
|
|
2655
2675
|
]);
|
|
2656
2676
|
var liveKitCapacitySnapshotSchema = z.object({
|
|
2657
2677
|
// Placement identity + per-worker session ceiling. These are always present on the
|
|
@@ -3918,6 +3938,7 @@ function useRealtimeSession(input) {
|
|
|
3918
3938
|
onEnded,
|
|
3919
3939
|
onBehaviorChange,
|
|
3920
3940
|
onClipResult,
|
|
3941
|
+
onVoiceError,
|
|
3921
3942
|
// everything else (client/active/idleSeconds/idleWarnLeadSeconds/reconnect/requestOptions…)
|
|
3922
3943
|
// flows straight into the inner SSOT hook.
|
|
3923
3944
|
...inner
|
|
@@ -3933,6 +3954,7 @@ function useRealtimeSession(input) {
|
|
|
3933
3954
|
const [turn, setTurn] = useState("quiet");
|
|
3934
3955
|
const [media, setMediaState] = useState({ video: "connecting", audio: "silent" });
|
|
3935
3956
|
const [behavior, setBehavior] = useState(null);
|
|
3957
|
+
const [voiceError, setVoiceError] = useState(null);
|
|
3936
3958
|
const [microphoneFacts2, setMicrophoneFactsState] = useState({
|
|
3937
3959
|
wanted: false,
|
|
3938
3960
|
connected: false,
|
|
@@ -3946,8 +3968,8 @@ function useRealtimeSession(input) {
|
|
|
3946
3968
|
const lastBehaviorRef = useRef(null);
|
|
3947
3969
|
const lastLabeledEndReasonRef = useRef(null);
|
|
3948
3970
|
const pendingClipsRef = useRef(/* @__PURE__ */ new Map());
|
|
3949
|
-
const cbRef = useRef({ onApproachingEnd, onGraceWindowOpen, onGraceWindowClosed, onIdleWarning, onCreditsLow, onTurnTimeout, onReconnecting, onReconnected, onEnded, onBehaviorChange, onClipResult });
|
|
3950
|
-
cbRef.current = { onApproachingEnd, onGraceWindowOpen, onGraceWindowClosed, onIdleWarning, onCreditsLow, onTurnTimeout, onReconnecting, onReconnected, onEnded, onBehaviorChange, onClipResult };
|
|
3971
|
+
const cbRef = useRef({ onApproachingEnd, onGraceWindowOpen, onGraceWindowClosed, onIdleWarning, onCreditsLow, onTurnTimeout, onReconnecting, onReconnected, onEnded, onBehaviorChange, onClipResult, onVoiceError });
|
|
3972
|
+
cbRef.current = { onApproachingEnd, onGraceWindowOpen, onGraceWindowClosed, onIdleWarning, onCreditsLow, onTurnTimeout, onReconnecting, onReconnected, onEnded, onBehaviorChange, onClipResult, onVoiceError };
|
|
3951
3973
|
const approachingEndLeadMs = positive(approachingEndLeadSeconds, DEFAULT_APPROACHING_END_LEAD_SECONDS) * 1e3;
|
|
3952
3974
|
const graceWindowLeadMs = positive(graceWindowLeadSeconds, DEFAULT_GRACE_WINDOW_LEAD_SECONDS) * 1e3;
|
|
3953
3975
|
const graceCeilingMs = positive(graceCeilingSeconds, DEFAULT_GRACE_CEILING_SECONDS) * 1e3;
|
|
@@ -4035,6 +4057,18 @@ function useRealtimeSession(input) {
|
|
|
4035
4057
|
cbRef.current.onClipResult?.(result);
|
|
4036
4058
|
break;
|
|
4037
4059
|
}
|
|
4060
|
+
case "voice_error": {
|
|
4061
|
+
const error = {
|
|
4062
|
+
code: f.code,
|
|
4063
|
+
message: f.message,
|
|
4064
|
+
...f.status === void 0 ? {} : { status: f.status },
|
|
4065
|
+
fatal: f.fatal,
|
|
4066
|
+
...f.request_id === void 0 ? {} : { requestId: f.request_id }
|
|
4067
|
+
};
|
|
4068
|
+
setVoiceError(error);
|
|
4069
|
+
cbRef.current.onVoiceError?.(error);
|
|
4070
|
+
break;
|
|
4071
|
+
}
|
|
4038
4072
|
}
|
|
4039
4073
|
}, []);
|
|
4040
4074
|
const registerDataPublisher = useCallback((publish) => {
|
|
@@ -4229,10 +4263,22 @@ function useRealtimeSession(input) {
|
|
|
4229
4263
|
if (active) return;
|
|
4230
4264
|
lastLabeledEndReasonRef.current = null;
|
|
4231
4265
|
setMicrophoneMuted(false);
|
|
4266
|
+
setVoiceError(null);
|
|
4232
4267
|
}, [active]);
|
|
4268
|
+
const grantSessionId = lifecycle.grant?.session_id ?? null;
|
|
4269
|
+
const voiceErrorGrantRef = useRef(null);
|
|
4270
|
+
useEffect(() => {
|
|
4271
|
+
if (phaseKind === "requesting") setVoiceError(null);
|
|
4272
|
+
if (grantSessionId === null) return;
|
|
4273
|
+
if (voiceErrorGrantRef.current !== null && voiceErrorGrantRef.current !== grantSessionId) setVoiceError(null);
|
|
4274
|
+
voiceErrorGrantRef.current = grantSessionId;
|
|
4275
|
+
}, [phaseKind, grantSessionId]);
|
|
4233
4276
|
const innerReconnect = lifecycle.reconnect;
|
|
4234
4277
|
const reconnect = useCallback(() => {
|
|
4235
|
-
if (phaseKind === "ended")
|
|
4278
|
+
if (phaseKind === "ended") {
|
|
4279
|
+
lastLabeledEndReasonRef.current = null;
|
|
4280
|
+
setVoiceError(null);
|
|
4281
|
+
}
|
|
4236
4282
|
innerReconnect();
|
|
4237
4283
|
}, [innerReconnect, phaseKind]);
|
|
4238
4284
|
const performAction = useCallback(
|
|
@@ -4281,6 +4327,7 @@ function useRealtimeSession(input) {
|
|
|
4281
4327
|
creditsLowFiredRef.current = false;
|
|
4282
4328
|
lastTurnRef.current = null;
|
|
4283
4329
|
setMicrophoneMuted(false);
|
|
4330
|
+
setVoiceError(null);
|
|
4284
4331
|
lifecycle.reset();
|
|
4285
4332
|
}, [lifecycle]);
|
|
4286
4333
|
return useMemo(
|
|
@@ -4309,6 +4356,7 @@ function useRealtimeSession(input) {
|
|
|
4309
4356
|
retryTurn,
|
|
4310
4357
|
end,
|
|
4311
4358
|
behavior,
|
|
4359
|
+
voiceError,
|
|
4312
4360
|
performAction,
|
|
4313
4361
|
onLifecycleData,
|
|
4314
4362
|
registerDataPublisher,
|
|
@@ -4341,6 +4389,7 @@ function useRealtimeSession(input) {
|
|
|
4341
4389
|
retryTurn,
|
|
4342
4390
|
end,
|
|
4343
4391
|
behavior,
|
|
4392
|
+
voiceError,
|
|
4344
4393
|
performAction,
|
|
4345
4394
|
onLifecycleData,
|
|
4346
4395
|
registerDataPublisher,
|
|
@@ -4387,7 +4436,8 @@ function handleFor(session) {
|
|
|
4387
4436
|
audio: session.audioPlayback,
|
|
4388
4437
|
startAudio: session.startAudio,
|
|
4389
4438
|
setMicrophoneEnabled: session.setMicrophoneEnabled,
|
|
4390
|
-
retryMicrophone: () => session.setMicrophoneEnabled(true)
|
|
4439
|
+
retryMicrophone: () => session.setMicrophoneEnabled(true),
|
|
4440
|
+
voiceError: session.voiceError
|
|
4391
4441
|
};
|
|
4392
4442
|
}
|
|
4393
4443
|
var UNLOCK_REGION = {
|
|
@@ -4419,7 +4469,8 @@ function useAvatarCall(props) {
|
|
|
4419
4469
|
} : void 0,
|
|
4420
4470
|
onIdleWarning: props.onQuiet ? ({ secondsLeft }) => props.onQuiet?.({ secondsLeft }) : void 0,
|
|
4421
4471
|
onCreditsLow: props.onLowBalance ? ({ secondsLeft }) => props.onLowBalance?.({ secondsLeft }) : void 0,
|
|
4422
|
-
onEnded: props.onEnded ? ({ reason }) => props.onEnded?.({ reason }) : void 0
|
|
4472
|
+
onEnded: props.onEnded ? ({ reason }) => props.onEnded?.({ reason }) : void 0,
|
|
4473
|
+
onVoiceError: props.onVoiceError ? (error) => props.onVoiceError?.(error) : void 0
|
|
4423
4474
|
});
|
|
4424
4475
|
sessionRef.current = session;
|
|
4425
4476
|
const call = handleFor(session);
|
package/dist/server.d.ts
CHANGED
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
export { RealtimeAvatar, RealtimeAvatarError, RealtimeAvatarHttpError, RealtimeAvatarOptions, StartCallOptions, clipLibraryDeclarationSchema, verifyTranscript } from './index.js';
|
|
2
|
-
export { h as Asset, g as AssetKind, A as Avatar, o as AvatarClip, c as AvatarSourceSwap, b as AvatarUpdate, p as CallConnection, C as CallMode, a as CallPolicy, q as CallQueued, r as ClipDeclaration, f as ClipLibrary, d as ClipLibraryDeclaration, s as ClipLibraryPlan, e as ClipLibraryUpdate, t as ClipSource, u as ContextMessage, n as CreditBalance, E as EndCallOptions, v as EndCallReason, w as ExternalLlm, k as ListRecordingsQuery, l as ListRecordingsResponse, i as ListSessionsOptions, L as LoopRedirect, m as RecordingAccessResponse, R as RecordingArtifact, x as RecordingArtifactStatus, y as RecordingMode, S as StartCallResult, T as TranscriptPayload, j as UsageSession, U as UsageSessionPage,
|
|
1
|
+
export { RealtimeAvatar, RealtimeAvatarError, RealtimeAvatarHttpError, RealtimeAvatarOptions, RealtimeAvatarWebhookError, StartCallOptions, WebhookVerificationErrorCode, clipLibraryDeclarationSchema, verifyTranscript, verifyVoiceWebhook } from './index.js';
|
|
2
|
+
export { h as Asset, g as AssetKind, A as Avatar, o as AvatarClip, c as AvatarSourceSwap, b as AvatarUpdate, p as CallConnection, C as CallMode, a as CallPolicy, q as CallQueued, r as ClipDeclaration, f as ClipLibrary, d as ClipLibraryDeclaration, s as ClipLibraryPlan, e as ClipLibraryUpdate, t as ClipSource, u as ContextMessage, n as CreditBalance, E as EndCallOptions, v as EndCallReason, w as ExternalLlm, k as ListRecordingsQuery, l as ListRecordingsResponse, i as ListSessionsOptions, L as LoopRedirect, m as RecordingAccessResponse, R as RecordingArtifact, x as RecordingArtifactStatus, y as RecordingMode, S as StartCallResult, T as TranscriptPayload, j as UsageSession, U as UsageSessionPage, z as VideoPolicy, B as VoiceWebhook, D as VoiceWebhookErrorCode, V as VoiceWebhookRequest, F as isQueued, G as listRecordingsResponseSchema, H as recordingAccessResponseSchema, I as recordingArtifactSchema, J as recordingModeSchema } from './types-Bv29uEAs.js';
|
|
3
3
|
import './connection-history.js';
|
|
4
4
|
import 'zod';
|
package/dist/server.js
CHANGED
|
@@ -1,2 +1,2 @@
|
|
|
1
|
-
export { RealtimeAvatar, RealtimeAvatarError, RealtimeAvatarHttpError, clipLibraryDeclarationSchema, isQueued, listRecordingsResponseSchema, recordingAccessResponseSchema, recordingArtifactSchema, recordingModeSchema, verifyTranscript } from './chunk-
|
|
1
|
+
export { RealtimeAvatar, RealtimeAvatarError, RealtimeAvatarHttpError, RealtimeAvatarWebhookError, clipLibraryDeclarationSchema, isQueued, listRecordingsResponseSchema, recordingAccessResponseSchema, recordingArtifactSchema, recordingModeSchema, verifyTranscript, verifyVoiceWebhook } from './chunk-MMO5AQN4.js';
|
|
2
2
|
import './chunk-MCIXU6XP.js';
|
package/dist/tanstack-start.d.ts
CHANGED
package/dist/tanstack-start.js
CHANGED