realtime-avatar 0.10.0 → 0.11.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/{chunk-UJQBYKPI.js → chunk-2TQVKQJJ.js} +1 -1
- package/dist/{chunk-5UEMZN6J.js → chunk-X3XXLAJJ.js} +1 -1
- package/dist/express.d.ts +2 -2
- package/dist/express.js +2 -2
- package/dist/hono.d.ts +2 -2
- package/dist/hono.js +2 -2
- package/dist/index.d.ts +2 -2
- package/dist/index.js +1 -1
- package/dist/nextjs.d.ts +2 -2
- package/dist/nextjs.js +2 -2
- package/dist/{proxy-client-B6OnXpaC.d.ts → proxy-client-D1Zm7reb.d.ts} +8 -11
- package/dist/react-native.d.ts +2 -2
- package/dist/react-native.js +63 -28
- package/dist/react.d.ts +12 -3
- package/dist/react.js +124 -36
- package/dist/server.d.ts +1 -1
- package/dist/server.js +1 -1
- package/dist/tanstack-start.d.ts +2 -2
- package/dist/tanstack-start.js +2 -2
- package/dist/{types-zQtYeR_l.d.ts → types-9a0bRdXl.d.ts} +1 -1
- package/dist/{types-BgYFwlSn.d.ts → types-nxdohaiH.d.ts} +5 -2
- package/package.json +1 -1
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
import { RealtimeAvatar, isQueued, RealtimeAvatarHttpError } from './chunk-
|
|
1
|
+
import { RealtimeAvatar, isQueued, RealtimeAvatarHttpError } from './chunk-X3XXLAJJ.js';
|
|
2
2
|
|
|
3
3
|
// ../proxy/src/config.ts
|
|
4
4
|
var json = (body, status = 200) => new Response(JSON.stringify(body), {
|
|
@@ -218,7 +218,7 @@ function isTransient(cause) {
|
|
|
218
218
|
|
|
219
219
|
// ../http-client/src/client.ts
|
|
220
220
|
var DEFAULT_BASE_URL = "https://realtimeavatar.ai/api/v1";
|
|
221
|
-
var SDK_VERSION = "0.
|
|
221
|
+
var SDK_VERSION = "0.11.1";
|
|
222
222
|
var RealtimeAvatar = class {
|
|
223
223
|
#apiKey;
|
|
224
224
|
#baseUrl;
|
package/dist/express.d.ts
CHANGED
package/dist/express.js
CHANGED
package/dist/hono.d.ts
CHANGED
package/dist/hono.js
CHANGED
package/dist/index.d.ts
CHANGED
|
@@ -1,5 +1,5 @@
|
|
|
1
|
-
import { a as CallPolicy, C as CallMode, S as StartCallResult, E as EndCallOptions, A as Avatar, b as AvatarUpdate, c as AvatarSourceSwap, d as ClipLibraryDeclaration, e as ClipLibraryUpdate, L as LoopRedirect, f as ClipLibrary, g as AssetKind, h as Asset, i as ListSessionsOptions, U as UsageSessionPage, j as UsageSession, k as CreditBalance, T as TranscriptPayload } from './types-
|
|
2
|
-
export { l as AvatarClip, m as CallConnection, n as CallQueued, o as ClipDeclaration, p as ClipLibraryPlan, q as ClipSource, r as ContextMessage, s as EndCallReason, V as VideoPolicy, t as isQueued } from './types-
|
|
1
|
+
import { a as CallPolicy, C as CallMode, S as StartCallResult, E as EndCallOptions, A as Avatar, b as AvatarUpdate, c as AvatarSourceSwap, d as ClipLibraryDeclaration, e as ClipLibraryUpdate, L as LoopRedirect, f as ClipLibrary, g as AssetKind, h as Asset, i as ListSessionsOptions, U as UsageSessionPage, j as UsageSession, k as CreditBalance, T as TranscriptPayload } from './types-nxdohaiH.js';
|
|
2
|
+
export { l as AvatarClip, m as CallConnection, n as CallQueued, o as ClipDeclaration, p as ClipLibraryPlan, q as ClipSource, r as ContextMessage, s as EndCallReason, V as VideoPolicy, t as isQueued } from './types-nxdohaiH.js';
|
|
3
3
|
import { z } from 'zod';
|
|
4
4
|
|
|
5
5
|
interface RealtimeAvatarOptions {
|
package/dist/index.js
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
export { RealtimeAvatar, RealtimeAvatarError, RealtimeAvatarHttpError, clipLibraryDeclarationSchema, isQueued, verifyTranscript } from './chunk-
|
|
1
|
+
export { RealtimeAvatar, RealtimeAvatarError, RealtimeAvatarHttpError, clipLibraryDeclarationSchema, isQueued, verifyTranscript } from './chunk-X3XXLAJJ.js';
|
package/dist/nextjs.d.ts
CHANGED
package/dist/nextjs.js
CHANGED
|
@@ -101,11 +101,6 @@ declare const sessionClipSchema: z.ZodObject<{
|
|
|
101
101
|
type SessionClip = z.infer<typeof sessionClipSchema>;
|
|
102
102
|
declare const sessionBehaviorSchema: z.ZodObject<{
|
|
103
103
|
gestures_enabled: z.ZodOptional<z.ZodBoolean>;
|
|
104
|
-
gesture_freq: z.ZodOptional<z.ZodEnum<{
|
|
105
|
-
sparse: "sparse";
|
|
106
|
-
balanced: "balanced";
|
|
107
|
-
lively: "lively";
|
|
108
|
-
}>>;
|
|
109
104
|
}, z.core.$strict>;
|
|
110
105
|
type SessionBehavior = z.infer<typeof sessionBehaviorSchema>;
|
|
111
106
|
/**
|
|
@@ -206,11 +201,6 @@ declare const liveKitSessionRequestSchema: z.ZodObject<{
|
|
|
206
201
|
}, z.core.$strict>>>;
|
|
207
202
|
behavior: z.ZodOptional<z.ZodObject<{
|
|
208
203
|
gestures_enabled: z.ZodOptional<z.ZodBoolean>;
|
|
209
|
-
gesture_freq: z.ZodOptional<z.ZodEnum<{
|
|
210
|
-
sparse: "sparse";
|
|
211
|
-
balanced: "balanced";
|
|
212
|
-
lively: "lively";
|
|
213
|
-
}>>;
|
|
214
204
|
}, z.core.$strict>>;
|
|
215
205
|
renderBackend: z.ZodOptional<z.ZodString>;
|
|
216
206
|
supportEdits: z.ZodOptional<z.ZodObject<{
|
|
@@ -478,7 +468,14 @@ type AvatarVideoSurfaceProps = {
|
|
|
478
468
|
idleReturnDelayMs?: number;
|
|
479
469
|
/**
|
|
480
470
|
* Maximum time without a newly presented decoded frame before the live layer is
|
|
481
|
-
* treated as stalled and
|
|
471
|
+
* treated as stalled and replaced by the idle/poster floor — i.e. how long the LAST
|
|
472
|
+
* live frame is held through a network gap. Default
|
|
473
|
+
* {@link DEFAULT_AVATAR_FRAME_STALL_MS} (2 s). Once two stalls land inside 15 s the
|
|
474
|
+
* link is treated as unstable and the hold steps up to
|
|
475
|
+
* {@link DEFAULT_AVATAR_UNSTABLE_STALL_MS} (4 s) until the window clears, so a
|
|
476
|
+
* flapping link settles on a briefly frozen face rather than a body swap per gap.
|
|
477
|
+
* A disconnect, a turn end (track muted) and a track that never presented a frame
|
|
478
|
+
* are not stalls and do not wait on this.
|
|
482
479
|
*/
|
|
483
480
|
frameStallMs?: number;
|
|
484
481
|
/** Extra className for the box (the layers fill it). */
|
package/dist/react-native.d.ts
CHANGED
|
@@ -1,8 +1,8 @@
|
|
|
1
1
|
import { LiveKitRoomProps } from '@livekit/react-native';
|
|
2
2
|
export { AudioSession, VideoTrack, VideoTrackProps, registerGlobals } from '@livekit/react-native';
|
|
3
3
|
import { ReactNode, ReactElement } from 'react';
|
|
4
|
-
import { L as LiveKitSessionGrant, A as AvatarVideoFit } from './proxy-client-
|
|
5
|
-
export { a as AdaptivePlayoutController, b as AdaptivePlayoutDecision, c as AdaptivePlayoutOptions, d as AdaptivePlayoutSample, e as ApproachingEndEvent, f as ApproachingEndReason, g as AvatarSessionClient, B as BehaviorSnapshot, C as CallTranscript, h as CallTranscriptSegment, i as CartesiaTtsModel, j as ClipResult, k as ClosingTurnResult, l as CreditsLowEvent, D as DEFAULT_APPROACHING_END_LEAD_SECONDS, m as DEFAULT_CREDITS_LOW_LEAD_SECONDS, n as DEFAULT_GOVERNOR_CONFIG, o as DEFAULT_GRACE_CEILING_SECONDS, p as DEFAULT_GRACE_WINDOW_LEAD_SECONDS, q as DEFAULT_IDLE_SECONDS, r as DEFAULT_IDLE_WARN_LEAD_SECONDS, s as DEFAULT_TURN_TIMEOUT_SECONDS, E as EndReason, t as EndedEvent, u as ExtendResult, F as FishTtsModel, v as FreezeReadingFn, G as Governor, w as GovernorAction, x as GovernorConfig, y as GovernorSignal, z as GovernorState, H as GraceWindowClosedEvent, I as GraceWindowOpenEvent, J as GraceWindowState, K as IdleWarningEvent, M as InboundRtpCursor, N as InboundRtpReading, O as KnownBehaviorState, P as LLMProvider, Q as LLMSelection, R as LiveKitAvatarGrantState, S as LiveKitAvatarGrantStatus, T as LiveKitCapacityState, U as LiveKitConnectionStatus, V as LiveKitSessionRequest, W as LiveKitSessionStartResult, X as MAX_RECONNECT_ATTEMPTS, Y as MAX_SESSION_INSTRUCTIONS_CHARS, Z as ProxyClientOptions, _ as QualityCap, $ as RECONNECT_BACKOFF_MS, a0 as RealtimeAvatarRequestOptions, a1 as RealtimeSessionApi, a2 as RealtimeSessionMedia, a3 as RealtimeSessionRoomSinks, a4 as ReconnectPolicy, a5 as ReconnectingEvent, a6 as RecoveryState, a7 as RetryStep, a8 as SessionBehavior, a9 as SessionClip, aa as SessionClocks, ab as SessionEndReason, ac as SessionLifecycleApi, ad as SessionLifecyclePhase, ae as SessionLifecyclePhaseKind, af as SessionLifecycleRoomBridge, ag as SessionLifecycleRoomBridgeProps, ah as SurfaceLayers, ai as TurnState, aj as TurnTimeoutEvent, ak as UseAvatarQualityGovernorInput, al as UseLiveKitAvatarGrantInput, am as UseRealtimeSessionInput, an as UseSessionLifecycleInput, ao as VoiceSpec, ap as VoiceSpecInput, aq as capacityErrorFromBusy, ar as capacityStateFromGrant, as as createProxyClient, at as isNativeLiveTrackSubscribed, au as knownBehaviorStates, av as mapTurnState, aw as readInboundRtp, ax as sessionBehaviorSchema, ay as sessionClipSchema, az as splitCallTranscript, aA as useAvatarAdaptivePlayoutDelay, aB as useAvatarPlayoutDelay, aC as useAvatarQualityGovernor, aD as useCallTranscript, aE as useLiveKitAvatarGrant, aF as useLiveTrackProducing, aG as useMicLease, aH as useRealtimeSession, aI as useReleaseMicLeaseOnTrackEnded, aJ as useSessionLifecycle } from './proxy-client-
|
|
4
|
+
import { L as LiveKitSessionGrant, A as AvatarVideoFit } from './proxy-client-D1Zm7reb.js';
|
|
5
|
+
export { a as AdaptivePlayoutController, b as AdaptivePlayoutDecision, c as AdaptivePlayoutOptions, d as AdaptivePlayoutSample, e as ApproachingEndEvent, f as ApproachingEndReason, g as AvatarSessionClient, B as BehaviorSnapshot, C as CallTranscript, h as CallTranscriptSegment, i as CartesiaTtsModel, j as ClipResult, k as ClosingTurnResult, l as CreditsLowEvent, D as DEFAULT_APPROACHING_END_LEAD_SECONDS, m as DEFAULT_CREDITS_LOW_LEAD_SECONDS, n as DEFAULT_GOVERNOR_CONFIG, o as DEFAULT_GRACE_CEILING_SECONDS, p as DEFAULT_GRACE_WINDOW_LEAD_SECONDS, q as DEFAULT_IDLE_SECONDS, r as DEFAULT_IDLE_WARN_LEAD_SECONDS, s as DEFAULT_TURN_TIMEOUT_SECONDS, E as EndReason, t as EndedEvent, u as ExtendResult, F as FishTtsModel, v as FreezeReadingFn, G as Governor, w as GovernorAction, x as GovernorConfig, y as GovernorSignal, z as GovernorState, H as GraceWindowClosedEvent, I as GraceWindowOpenEvent, J as GraceWindowState, K as IdleWarningEvent, M as InboundRtpCursor, N as InboundRtpReading, O as KnownBehaviorState, P as LLMProvider, Q as LLMSelection, R as LiveKitAvatarGrantState, S as LiveKitAvatarGrantStatus, T as LiveKitCapacityState, U as LiveKitConnectionStatus, V as LiveKitSessionRequest, W as LiveKitSessionStartResult, X as MAX_RECONNECT_ATTEMPTS, Y as MAX_SESSION_INSTRUCTIONS_CHARS, Z as ProxyClientOptions, _ as QualityCap, $ as RECONNECT_BACKOFF_MS, a0 as RealtimeAvatarRequestOptions, a1 as RealtimeSessionApi, a2 as RealtimeSessionMedia, a3 as RealtimeSessionRoomSinks, a4 as ReconnectPolicy, a5 as ReconnectingEvent, a6 as RecoveryState, a7 as RetryStep, a8 as SessionBehavior, a9 as SessionClip, aa as SessionClocks, ab as SessionEndReason, ac as SessionLifecycleApi, ad as SessionLifecyclePhase, ae as SessionLifecyclePhaseKind, af as SessionLifecycleRoomBridge, ag as SessionLifecycleRoomBridgeProps, ah as SurfaceLayers, ai as TurnState, aj as TurnTimeoutEvent, ak as UseAvatarQualityGovernorInput, al as UseLiveKitAvatarGrantInput, am as UseRealtimeSessionInput, an as UseSessionLifecycleInput, ao as VoiceSpec, ap as VoiceSpecInput, aq as capacityErrorFromBusy, ar as capacityStateFromGrant, as as createProxyClient, at as isNativeLiveTrackSubscribed, au as knownBehaviorStates, av as mapTurnState, aw as readInboundRtp, ax as sessionBehaviorSchema, ay as sessionClipSchema, az as splitCallTranscript, aA as useAvatarAdaptivePlayoutDelay, aB as useAvatarPlayoutDelay, aC as useAvatarQualityGovernor, aD as useCallTranscript, aE as useLiveKitAvatarGrant, aF as useLiveTrackProducing, aG as useMicLease, aH as useRealtimeSession, aI as useReleaseMicLeaseOnTrackEnded, aJ as useSessionLifecycle } from './proxy-client-D1Zm7reb.js';
|
|
6
6
|
import { StyleProp, ViewStyle } from 'react-native';
|
|
7
7
|
export { useChat, useConnectionState, useLocalParticipant, useRoomContext, useTrackToggle, useTranscriptions, useVoiceAssistant } from '@livekit/components-react';
|
|
8
8
|
export { D as DEFAULT_AVATAR_PLAYOUT_DELAY_SECONDS } from './playout-delay-DWOc8iCl.js';
|
package/dist/react-native.js
CHANGED
|
@@ -858,38 +858,74 @@ var qualityToSignal = (q) => {
|
|
|
858
858
|
}
|
|
859
859
|
};
|
|
860
860
|
function useAvatarQualityGovernor(input) {
|
|
861
|
-
const { enabled, freezeReading, config = DEFAULT_GOVERNOR_CONFIG, tickMs = 1e3 } = input;
|
|
861
|
+
const { enabled, freezeReading, config: policy = DEFAULT_GOVERNOR_CONFIG, tickMs = 1e3 } = input;
|
|
862
862
|
const room = useMaybeRoomContext();
|
|
863
863
|
const { videoTrack } = useVoiceAssistant();
|
|
864
|
-
const
|
|
865
|
-
|
|
866
|
-
|
|
867
|
-
|
|
864
|
+
const {
|
|
865
|
+
openingCap,
|
|
866
|
+
downgradeFreezeMs,
|
|
867
|
+
probationFreezeMs,
|
|
868
|
+
openingDwellMs,
|
|
869
|
+
dwellBaseMs,
|
|
870
|
+
dwellMaxMs,
|
|
871
|
+
cleanMs,
|
|
872
|
+
probeMs,
|
|
873
|
+
healthyResetMs
|
|
874
|
+
} = policy;
|
|
875
|
+
const config = useMemo(() => ({
|
|
876
|
+
openingCap,
|
|
877
|
+
downgradeFreezeMs,
|
|
878
|
+
probationFreezeMs,
|
|
879
|
+
openingDwellMs,
|
|
880
|
+
dwellBaseMs,
|
|
881
|
+
dwellMaxMs,
|
|
882
|
+
cleanMs,
|
|
883
|
+
probeMs,
|
|
884
|
+
healthyResetMs
|
|
885
|
+
}), [
|
|
886
|
+
openingCap,
|
|
887
|
+
downgradeFreezeMs,
|
|
888
|
+
probationFreezeMs,
|
|
889
|
+
openingDwellMs,
|
|
890
|
+
dwellBaseMs,
|
|
891
|
+
dwellMaxMs,
|
|
892
|
+
cleanMs,
|
|
893
|
+
probeMs,
|
|
894
|
+
healthyResetMs
|
|
895
|
+
]);
|
|
896
|
+
const freezeReadingRef = useRef(freezeReading);
|
|
897
|
+
freezeReadingRef.current = freezeReading;
|
|
898
|
+
const targetPublication = videoTrack?.publication;
|
|
899
|
+
const targetParticipant = videoTrack?.participant;
|
|
900
|
+
const targetTrack = targetPublication?.track;
|
|
868
901
|
useEffect(() => {
|
|
869
|
-
if (!enabled || !room) return;
|
|
870
|
-
|
|
871
|
-
|
|
872
|
-
|
|
873
|
-
|
|
874
|
-
jitterTrend.current = null;
|
|
875
|
-
connQuality.current = qualityToSignal(
|
|
902
|
+
if (!enabled || !room || !targetPublication || !targetParticipant) return;
|
|
903
|
+
let pausedSinceTick = false;
|
|
904
|
+
let lastFreezeStat = null;
|
|
905
|
+
let jitterTrend = null;
|
|
906
|
+
let connQuality = qualityToSignal(
|
|
876
907
|
targetParticipant?.connectionQuality ?? ConnectionQuality.Unknown
|
|
877
908
|
);
|
|
878
909
|
const onStreamState = (pub, streamState) => {
|
|
879
|
-
if (
|
|
880
|
-
if (streamState === Track.StreamState.Paused) pausedSinceTick
|
|
910
|
+
if (pub !== targetPublication) return;
|
|
911
|
+
if (streamState === Track.StreamState.Paused) pausedSinceTick = true;
|
|
881
912
|
};
|
|
882
913
|
const onQuality = (q, participant) => {
|
|
883
|
-
if (
|
|
884
|
-
connQuality
|
|
914
|
+
if (participant.sid !== targetParticipant.sid) return;
|
|
915
|
+
connQuality = qualityToSignal(q);
|
|
885
916
|
};
|
|
886
917
|
try {
|
|
887
918
|
room.on(RoomEvent.TrackStreamStateChanged, onStreamState);
|
|
888
919
|
room.on(RoomEvent.ConnectionQualityChanged, onQuality);
|
|
889
920
|
} catch {
|
|
921
|
+
try {
|
|
922
|
+
room.off(RoomEvent.TrackStreamStateChanged, onStreamState);
|
|
923
|
+
room.off(RoomEvent.ConnectionQualityChanged, onQuality);
|
|
924
|
+
} catch {
|
|
925
|
+
}
|
|
890
926
|
return;
|
|
891
927
|
}
|
|
892
|
-
let gov = initGovernor(Date.now());
|
|
928
|
+
let gov = initGovernor(Date.now(), config.openingCap);
|
|
893
929
|
const readGetStatsSignals = async () => {
|
|
894
930
|
try {
|
|
895
931
|
const track = targetPublication?.track;
|
|
@@ -908,13 +944,13 @@ function useAvatarQualityGovernor(input) {
|
|
|
908
944
|
}
|
|
909
945
|
});
|
|
910
946
|
const now = Date.now();
|
|
911
|
-
const prev = lastFreezeStat
|
|
912
|
-
lastFreezeStat
|
|
913
|
-
const trend = stepJitterBufferTrend(jitterTrend
|
|
947
|
+
const prev = lastFreezeStat;
|
|
948
|
+
lastFreezeStat = { frozen: frozenTotalMs, ts: now };
|
|
949
|
+
const trend = stepJitterBufferTrend(jitterTrend, {
|
|
914
950
|
delaySeconds: jitterDelaySeconds,
|
|
915
951
|
emittedCount: jitterEmittedCount
|
|
916
952
|
});
|
|
917
|
-
jitterTrend
|
|
953
|
+
jitterTrend = trend.state;
|
|
918
954
|
return {
|
|
919
955
|
freezeMs: prev ? Math.max(0, frozenTotalMs - prev.frozen) : 0,
|
|
920
956
|
jitterRising: trend.rising
|
|
@@ -938,17 +974,17 @@ function useAvatarQualityGovernor(input) {
|
|
|
938
974
|
if (disposed || tickRunning) return;
|
|
939
975
|
tickRunning = true;
|
|
940
976
|
try {
|
|
941
|
-
const rvfc =
|
|
977
|
+
const rvfc = freezeReadingRef.current?.() ?? { freezeMsInWindow: 0, inhibited: false };
|
|
942
978
|
const statsSignals = await readGetStatsSignals();
|
|
943
979
|
if (disposed) return;
|
|
944
980
|
const signal = {
|
|
945
|
-
paused: pausedSinceTick
|
|
981
|
+
paused: pausedSinceTick,
|
|
946
982
|
freezeMsInWindow: Math.max(rvfc.freezeMsInWindow, statsSignals.freezeMs),
|
|
947
983
|
jitterRising: statsSignals.jitterRising,
|
|
948
|
-
connectionQuality: connQuality
|
|
984
|
+
connectionQuality: connQuality,
|
|
949
985
|
inhibited: rvfc.inhibited
|
|
950
986
|
};
|
|
951
|
-
pausedSinceTick
|
|
987
|
+
pausedSinceTick = false;
|
|
952
988
|
const { governor, action } = step(gov, signal, Date.now(), config);
|
|
953
989
|
gov = governor;
|
|
954
990
|
if (action) applyCap(action.setCap);
|
|
@@ -968,7 +1004,7 @@ function useAvatarQualityGovernor(input) {
|
|
|
968
1004
|
} catch {
|
|
969
1005
|
}
|
|
970
1006
|
};
|
|
971
|
-
}, [enabled, room,
|
|
1007
|
+
}, [enabled, room, targetPublication, targetParticipant, targetTrack, config, tickMs]);
|
|
972
1008
|
}
|
|
973
1009
|
|
|
974
1010
|
// ../client/src/react/avatar-video-surface.ts
|
|
@@ -1297,8 +1333,7 @@ var sessionChoreographySchema = z.object({
|
|
|
1297
1333
|
{ message: "idle_dwell_min_seconds must be <= idle_dwell_max_seconds" }
|
|
1298
1334
|
);
|
|
1299
1335
|
var sessionBehaviorSchema = z.object({
|
|
1300
|
-
gestures_enabled: z.boolean().optional()
|
|
1301
|
-
gesture_freq: z.enum(["sparse", "balanced", "lively"]).optional()
|
|
1336
|
+
gestures_enabled: z.boolean().optional()
|
|
1302
1337
|
}).strict();
|
|
1303
1338
|
var transcriptWebhookSchema = z.object({
|
|
1304
1339
|
url: z.string().url().max(500),
|
package/dist/react.d.ts
CHANGED
|
@@ -1,12 +1,21 @@
|
|
|
1
1
|
import { RegisteredTool, ToolRegistration } from './tools.js';
|
|
2
2
|
import { ReactNode } from 'react';
|
|
3
|
-
import { g as AvatarSessionClient, A as AvatarVideoFit } from './proxy-client-
|
|
4
|
-
export { a as AdaptivePlayoutController, b as AdaptivePlayoutDecision, c as AdaptivePlayoutOptions, d as AdaptivePlayoutSample, e as ApproachingEndEvent, f as ApproachingEndReason, aK as AvatarVideoSurface, aL as AvatarVideoSurfaceProps, B as BehaviorSnapshot, C as CallTranscript, h as CallTranscriptSegment, i as CartesiaTtsModel, j as ClipResult, k as ClosingTurnResult, l as CreditsLowEvent, D as DEFAULT_APPROACHING_END_LEAD_SECONDS, m as DEFAULT_CREDITS_LOW_LEAD_SECONDS, n as DEFAULT_GOVERNOR_CONFIG, o as DEFAULT_GRACE_CEILING_SECONDS, p as DEFAULT_GRACE_WINDOW_LEAD_SECONDS, q as DEFAULT_IDLE_SECONDS, r as DEFAULT_IDLE_WARN_LEAD_SECONDS, s as DEFAULT_TURN_TIMEOUT_SECONDS, E as EndReason, t as EndedEvent, u as ExtendResult, F as FishTtsModel, v as FreezeReadingFn, G as Governor, w as GovernorAction, x as GovernorConfig, y as GovernorSignal, z as GovernorState, H as GraceWindowClosedEvent, I as GraceWindowOpenEvent, J as GraceWindowState, K as IdleWarningEvent, M as InboundRtpCursor, N as InboundRtpReading, O as KnownBehaviorState, P as LLMProvider, Q as LLMSelection, R as LiveKitAvatarGrantState, S as LiveKitAvatarGrantStatus, T as LiveKitCapacityState, U as LiveKitConnectionStatus, V as LiveKitSessionRequest, W as LiveKitSessionStartResult, aM as LivePlaybackKeeper, X as MAX_RECONNECT_ATTEMPTS, Y as MAX_SESSION_INSTRUCTIONS_CHARS, aN as MIC_LEASE_ENDED_TIMEOUT_MS, aO as MicLease, aP as PlayableVideoElement, Z as ProxyClientOptions, _ as QualityCap, $ as RECONNECT_BACKOFF_MS, aQ as RealtimeAvatarCapacityError, aR as RealtimeAvatarLiveKitRoom, aS as RealtimeAvatarLiveKitRoomProps, a0 as RealtimeAvatarRequestOptions, a1 as RealtimeSessionApi, a2 as RealtimeSessionMedia, a3 as RealtimeSessionRoomSinks, a4 as ReconnectPolicy, a5 as ReconnectingEvent, a6 as RecoveryState, a7 as RetryStep, a8 as SessionBehavior, a9 as SessionClip, aa as SessionClocks, ab as SessionEndReason, ac as SessionLifecycleApi, ad as SessionLifecyclePhase, ae as SessionLifecyclePhaseKind, af as SessionLifecycleRoomBridge, ag as SessionLifecycleRoomBridgeProps, ah as SurfaceLayers, ai as TurnState, aj as TurnTimeoutEvent, ak as UseAvatarQualityGovernorInput, al as UseLiveKitAvatarGrantInput, am as UseRealtimeSessionInput, an as UseSessionLifecycleInput, ao as VoiceSpec, ap as VoiceSpecInput, aq as capacityErrorFromBusy, ar as capacityStateFromGrant, as as createProxyClient, au as knownBehaviorStates, av as mapTurnState, aw as readInboundRtp, ax as sessionBehaviorSchema, ay as sessionClipSchema, aT as shouldReplayPendingTurn, az as splitCallTranscript, aA as useAvatarAdaptivePlayoutDelay, aB as useAvatarPlayoutDelay, aC as useAvatarQualityGovernor, aD as useCallTranscript, aE as useLiveKitAvatarGrant, aG as useMicLease, aH as useRealtimeSession, aI as useReleaseMicLeaseOnTrackEnded, aJ as useSessionLifecycle } from './proxy-client-
|
|
3
|
+
import { g as AvatarSessionClient, A as AvatarVideoFit } from './proxy-client-D1Zm7reb.js';
|
|
4
|
+
export { a as AdaptivePlayoutController, b as AdaptivePlayoutDecision, c as AdaptivePlayoutOptions, d as AdaptivePlayoutSample, e as ApproachingEndEvent, f as ApproachingEndReason, aK as AvatarVideoSurface, aL as AvatarVideoSurfaceProps, B as BehaviorSnapshot, C as CallTranscript, h as CallTranscriptSegment, i as CartesiaTtsModel, j as ClipResult, k as ClosingTurnResult, l as CreditsLowEvent, D as DEFAULT_APPROACHING_END_LEAD_SECONDS, m as DEFAULT_CREDITS_LOW_LEAD_SECONDS, n as DEFAULT_GOVERNOR_CONFIG, o as DEFAULT_GRACE_CEILING_SECONDS, p as DEFAULT_GRACE_WINDOW_LEAD_SECONDS, q as DEFAULT_IDLE_SECONDS, r as DEFAULT_IDLE_WARN_LEAD_SECONDS, s as DEFAULT_TURN_TIMEOUT_SECONDS, E as EndReason, t as EndedEvent, u as ExtendResult, F as FishTtsModel, v as FreezeReadingFn, G as Governor, w as GovernorAction, x as GovernorConfig, y as GovernorSignal, z as GovernorState, H as GraceWindowClosedEvent, I as GraceWindowOpenEvent, J as GraceWindowState, K as IdleWarningEvent, M as InboundRtpCursor, N as InboundRtpReading, O as KnownBehaviorState, P as LLMProvider, Q as LLMSelection, R as LiveKitAvatarGrantState, S as LiveKitAvatarGrantStatus, T as LiveKitCapacityState, U as LiveKitConnectionStatus, V as LiveKitSessionRequest, W as LiveKitSessionStartResult, aM as LivePlaybackKeeper, X as MAX_RECONNECT_ATTEMPTS, Y as MAX_SESSION_INSTRUCTIONS_CHARS, aN as MIC_LEASE_ENDED_TIMEOUT_MS, aO as MicLease, aP as PlayableVideoElement, Z as ProxyClientOptions, _ as QualityCap, $ as RECONNECT_BACKOFF_MS, aQ as RealtimeAvatarCapacityError, aR as RealtimeAvatarLiveKitRoom, aS as RealtimeAvatarLiveKitRoomProps, a0 as RealtimeAvatarRequestOptions, a1 as RealtimeSessionApi, a2 as RealtimeSessionMedia, a3 as RealtimeSessionRoomSinks, a4 as ReconnectPolicy, a5 as ReconnectingEvent, a6 as RecoveryState, a7 as RetryStep, a8 as SessionBehavior, a9 as SessionClip, aa as SessionClocks, ab as SessionEndReason, ac as SessionLifecycleApi, ad as SessionLifecyclePhase, ae as SessionLifecyclePhaseKind, af as SessionLifecycleRoomBridge, ag as SessionLifecycleRoomBridgeProps, ah as SurfaceLayers, ai as TurnState, aj as TurnTimeoutEvent, ak as UseAvatarQualityGovernorInput, al as UseLiveKitAvatarGrantInput, am as UseRealtimeSessionInput, an as UseSessionLifecycleInput, ao as VoiceSpec, ap as VoiceSpecInput, aq as capacityErrorFromBusy, ar as capacityStateFromGrant, as as createProxyClient, au as knownBehaviorStates, av as mapTurnState, aw as readInboundRtp, ax as sessionBehaviorSchema, ay as sessionClipSchema, aT as shouldReplayPendingTurn, az as splitCallTranscript, aA as useAvatarAdaptivePlayoutDelay, aB as useAvatarPlayoutDelay, aC as useAvatarQualityGovernor, aD as useCallTranscript, aE as useLiveKitAvatarGrant, aG as useMicLease, aH as useRealtimeSession, aI as useReleaseMicLeaseOnTrackEnded, aJ as useSessionLifecycle } from './proxy-client-D1Zm7reb.js';
|
|
5
5
|
export { D as DEFAULT_AVATAR_PLAYOUT_DELAY_SECONDS } from './playout-delay-DWOc8iCl.js';
|
|
6
6
|
import 'zod';
|
|
7
7
|
import '@livekit/components-react';
|
|
8
8
|
import 'livekit-client';
|
|
9
9
|
|
|
10
|
+
/**
|
|
11
|
+
* Once a link has proven unstable, hold the frozen live frame this long before
|
|
12
|
+
* falling back to the idle floor. Measured on a prod rtx6000 call through a
|
|
13
|
+
* 900 kbit / 150 ms / 10 % loss link (2026-09-09): after the opening, presented-frame
|
|
14
|
+
* gaps cluster at 650–1150 ms with two multi-second outages around simulcast layer
|
|
15
|
+
* switches. Nothing shorter than a few seconds is worth a body swap on such a link.
|
|
16
|
+
*/
|
|
17
|
+
declare const DEFAULT_AVATAR_UNSTABLE_STALL_MS = 4000;
|
|
18
|
+
|
|
10
19
|
declare function useCharacterTools(tools: Record<string, RegisteredTool>, options?: {
|
|
11
20
|
onResult?: (event: {
|
|
12
21
|
name: string;
|
|
@@ -120,4 +129,4 @@ declare function useAvatarCall(props: AvatarCallProps): {
|
|
|
120
129
|
/** One component: a live character on screen. */
|
|
121
130
|
declare function AvatarCall(props: AvatarCallProps): ReactNode;
|
|
122
131
|
|
|
123
|
-
export { AvatarCall, type AvatarCallEndReason, type AvatarCallHandle, type AvatarCallProps, type AvatarCallStatus, AvatarSessionClient, AvatarVideoFit, useAvatarCall, useCharacterTools };
|
|
132
|
+
export { AvatarCall, type AvatarCallEndReason, type AvatarCallHandle, type AvatarCallProps, type AvatarCallStatus, AvatarSessionClient, AvatarVideoFit, DEFAULT_AVATAR_UNSTABLE_STALL_MS, useAvatarCall, useCharacterTools };
|
package/dist/react.js
CHANGED
|
@@ -1059,38 +1059,74 @@ var qualityToSignal = (q) => {
|
|
|
1059
1059
|
}
|
|
1060
1060
|
};
|
|
1061
1061
|
function useAvatarQualityGovernor(input) {
|
|
1062
|
-
const { enabled, freezeReading, config = DEFAULT_GOVERNOR_CONFIG, tickMs = 1e3 } = input;
|
|
1062
|
+
const { enabled, freezeReading, config: policy = DEFAULT_GOVERNOR_CONFIG, tickMs = 1e3 } = input;
|
|
1063
1063
|
const room = useMaybeRoomContext();
|
|
1064
1064
|
const { videoTrack } = useVoiceAssistant();
|
|
1065
|
-
const
|
|
1066
|
-
|
|
1067
|
-
|
|
1068
|
-
|
|
1065
|
+
const {
|
|
1066
|
+
openingCap,
|
|
1067
|
+
downgradeFreezeMs,
|
|
1068
|
+
probationFreezeMs,
|
|
1069
|
+
openingDwellMs,
|
|
1070
|
+
dwellBaseMs,
|
|
1071
|
+
dwellMaxMs,
|
|
1072
|
+
cleanMs,
|
|
1073
|
+
probeMs,
|
|
1074
|
+
healthyResetMs
|
|
1075
|
+
} = policy;
|
|
1076
|
+
const config = useMemo(() => ({
|
|
1077
|
+
openingCap,
|
|
1078
|
+
downgradeFreezeMs,
|
|
1079
|
+
probationFreezeMs,
|
|
1080
|
+
openingDwellMs,
|
|
1081
|
+
dwellBaseMs,
|
|
1082
|
+
dwellMaxMs,
|
|
1083
|
+
cleanMs,
|
|
1084
|
+
probeMs,
|
|
1085
|
+
healthyResetMs
|
|
1086
|
+
}), [
|
|
1087
|
+
openingCap,
|
|
1088
|
+
downgradeFreezeMs,
|
|
1089
|
+
probationFreezeMs,
|
|
1090
|
+
openingDwellMs,
|
|
1091
|
+
dwellBaseMs,
|
|
1092
|
+
dwellMaxMs,
|
|
1093
|
+
cleanMs,
|
|
1094
|
+
probeMs,
|
|
1095
|
+
healthyResetMs
|
|
1096
|
+
]);
|
|
1097
|
+
const freezeReadingRef = useRef(freezeReading);
|
|
1098
|
+
freezeReadingRef.current = freezeReading;
|
|
1099
|
+
const targetPublication = videoTrack?.publication;
|
|
1100
|
+
const targetParticipant = videoTrack?.participant;
|
|
1101
|
+
const targetTrack = targetPublication?.track;
|
|
1069
1102
|
useEffect(() => {
|
|
1070
|
-
if (!enabled || !room) return;
|
|
1071
|
-
|
|
1072
|
-
|
|
1073
|
-
|
|
1074
|
-
|
|
1075
|
-
jitterTrend.current = null;
|
|
1076
|
-
connQuality.current = qualityToSignal(
|
|
1103
|
+
if (!enabled || !room || !targetPublication || !targetParticipant) return;
|
|
1104
|
+
let pausedSinceTick = false;
|
|
1105
|
+
let lastFreezeStat = null;
|
|
1106
|
+
let jitterTrend = null;
|
|
1107
|
+
let connQuality = qualityToSignal(
|
|
1077
1108
|
targetParticipant?.connectionQuality ?? ConnectionQuality.Unknown
|
|
1078
1109
|
);
|
|
1079
1110
|
const onStreamState = (pub, streamState) => {
|
|
1080
|
-
if (
|
|
1081
|
-
if (streamState === Track.StreamState.Paused) pausedSinceTick
|
|
1111
|
+
if (pub !== targetPublication) return;
|
|
1112
|
+
if (streamState === Track.StreamState.Paused) pausedSinceTick = true;
|
|
1082
1113
|
};
|
|
1083
1114
|
const onQuality = (q, participant) => {
|
|
1084
|
-
if (
|
|
1085
|
-
connQuality
|
|
1115
|
+
if (participant.sid !== targetParticipant.sid) return;
|
|
1116
|
+
connQuality = qualityToSignal(q);
|
|
1086
1117
|
};
|
|
1087
1118
|
try {
|
|
1088
1119
|
room.on(RoomEvent.TrackStreamStateChanged, onStreamState);
|
|
1089
1120
|
room.on(RoomEvent.ConnectionQualityChanged, onQuality);
|
|
1090
1121
|
} catch {
|
|
1122
|
+
try {
|
|
1123
|
+
room.off(RoomEvent.TrackStreamStateChanged, onStreamState);
|
|
1124
|
+
room.off(RoomEvent.ConnectionQualityChanged, onQuality);
|
|
1125
|
+
} catch {
|
|
1126
|
+
}
|
|
1091
1127
|
return;
|
|
1092
1128
|
}
|
|
1093
|
-
let gov = initGovernor(Date.now());
|
|
1129
|
+
let gov = initGovernor(Date.now(), config.openingCap);
|
|
1094
1130
|
const readGetStatsSignals = async () => {
|
|
1095
1131
|
try {
|
|
1096
1132
|
const track = targetPublication?.track;
|
|
@@ -1109,13 +1145,13 @@ function useAvatarQualityGovernor(input) {
|
|
|
1109
1145
|
}
|
|
1110
1146
|
});
|
|
1111
1147
|
const now = Date.now();
|
|
1112
|
-
const prev = lastFreezeStat
|
|
1113
|
-
lastFreezeStat
|
|
1114
|
-
const trend = stepJitterBufferTrend(jitterTrend
|
|
1148
|
+
const prev = lastFreezeStat;
|
|
1149
|
+
lastFreezeStat = { frozen: frozenTotalMs, ts: now };
|
|
1150
|
+
const trend = stepJitterBufferTrend(jitterTrend, {
|
|
1115
1151
|
delaySeconds: jitterDelaySeconds,
|
|
1116
1152
|
emittedCount: jitterEmittedCount
|
|
1117
1153
|
});
|
|
1118
|
-
jitterTrend
|
|
1154
|
+
jitterTrend = trend.state;
|
|
1119
1155
|
return {
|
|
1120
1156
|
freezeMs: prev ? Math.max(0, frozenTotalMs - prev.frozen) : 0,
|
|
1121
1157
|
jitterRising: trend.rising
|
|
@@ -1139,17 +1175,17 @@ function useAvatarQualityGovernor(input) {
|
|
|
1139
1175
|
if (disposed || tickRunning) return;
|
|
1140
1176
|
tickRunning = true;
|
|
1141
1177
|
try {
|
|
1142
|
-
const rvfc =
|
|
1178
|
+
const rvfc = freezeReadingRef.current?.() ?? { freezeMsInWindow: 0, inhibited: false };
|
|
1143
1179
|
const statsSignals = await readGetStatsSignals();
|
|
1144
1180
|
if (disposed) return;
|
|
1145
1181
|
const signal = {
|
|
1146
|
-
paused: pausedSinceTick
|
|
1182
|
+
paused: pausedSinceTick,
|
|
1147
1183
|
freezeMsInWindow: Math.max(rvfc.freezeMsInWindow, statsSignals.freezeMs),
|
|
1148
1184
|
jitterRising: statsSignals.jitterRising,
|
|
1149
|
-
connectionQuality: connQuality
|
|
1185
|
+
connectionQuality: connQuality,
|
|
1150
1186
|
inhibited: rvfc.inhibited
|
|
1151
1187
|
};
|
|
1152
|
-
pausedSinceTick
|
|
1188
|
+
pausedSinceTick = false;
|
|
1153
1189
|
const { governor, action } = step(gov, signal, Date.now(), config);
|
|
1154
1190
|
gov = governor;
|
|
1155
1191
|
if (action) applyCap(action.setCap);
|
|
@@ -1169,7 +1205,7 @@ function useAvatarQualityGovernor(input) {
|
|
|
1169
1205
|
} catch {
|
|
1170
1206
|
}
|
|
1171
1207
|
};
|
|
1172
|
-
}, [enabled, room,
|
|
1208
|
+
}, [enabled, room, targetPublication, targetParticipant, targetTrack, config, tickMs]);
|
|
1173
1209
|
}
|
|
1174
1210
|
|
|
1175
1211
|
// ../client/src/react/frame-recovery.ts
|
|
@@ -1198,9 +1234,48 @@ var FrameRecovery = class {
|
|
|
1198
1234
|
return this.ready;
|
|
1199
1235
|
}
|
|
1200
1236
|
};
|
|
1237
|
+
var AVATAR_FIRST_FRAME_GRACE_MS = 1e3;
|
|
1238
|
+
function firstFrameWaitFreezeMs(waitedMs) {
|
|
1239
|
+
if (!Number.isFinite(waitedMs) || waitedMs <= AVATAR_FIRST_FRAME_GRACE_MS) return 0;
|
|
1240
|
+
return waitedMs - AVATAR_FIRST_FRAME_GRACE_MS;
|
|
1241
|
+
}
|
|
1242
|
+
var DEFAULT_AVATAR_UNSTABLE_STALL_MS = 4e3;
|
|
1243
|
+
var AVATAR_UNSTABLE_LINK_STALLS = 2;
|
|
1244
|
+
var AVATAR_UNSTABLE_LINK_WINDOW_MS = 15e3;
|
|
1245
|
+
var StallEscalation = class {
|
|
1246
|
+
constructor(baseMs, unstableMs = DEFAULT_AVATAR_UNSTABLE_STALL_MS, windowMs = AVATAR_UNSTABLE_LINK_WINDOW_MS, stallsToEscalate = AVATAR_UNSTABLE_LINK_STALLS) {
|
|
1247
|
+
this.baseMs = baseMs;
|
|
1248
|
+
this.windowMs = windowMs;
|
|
1249
|
+
this.stallsToEscalate = stallsToEscalate;
|
|
1250
|
+
this.unstableMs = Math.max(baseMs, unstableMs);
|
|
1251
|
+
}
|
|
1252
|
+
baseMs;
|
|
1253
|
+
windowMs;
|
|
1254
|
+
stallsToEscalate;
|
|
1255
|
+
stallsAtMs = [];
|
|
1256
|
+
unstableMs;
|
|
1257
|
+
/** Record one stall EPISODE (the moment the live layer is hidden), not one poll tick. */
|
|
1258
|
+
recordStall(nowMs) {
|
|
1259
|
+
this.prune(nowMs);
|
|
1260
|
+
this.stallsAtMs.push(nowMs);
|
|
1261
|
+
}
|
|
1262
|
+
/** The threshold in force right now. */
|
|
1263
|
+
thresholdMs(nowMs) {
|
|
1264
|
+
this.prune(nowMs);
|
|
1265
|
+
return this.stallsAtMs.length >= this.stallsToEscalate ? this.unstableMs : this.baseMs;
|
|
1266
|
+
}
|
|
1267
|
+
/** Is the link currently considered unstable (escalated)? */
|
|
1268
|
+
unstable(nowMs) {
|
|
1269
|
+
return this.thresholdMs(nowMs) !== this.baseMs;
|
|
1270
|
+
}
|
|
1271
|
+
prune(nowMs) {
|
|
1272
|
+
const cutoff = nowMs - this.windowMs;
|
|
1273
|
+
while (this.stallsAtMs.length > 0 && this.stallsAtMs[0] < cutoff) this.stallsAtMs.shift();
|
|
1274
|
+
}
|
|
1275
|
+
};
|
|
1201
1276
|
|
|
1202
1277
|
// ../client/src/react/avatar-video-surface.ts
|
|
1203
|
-
var DEFAULT_AVATAR_FRAME_STALL_MS =
|
|
1278
|
+
var DEFAULT_AVATAR_FRAME_STALL_MS = 2e3;
|
|
1204
1279
|
var AVATAR_FRAME_GAP_FREEZE_FLOOR_MS = 100;
|
|
1205
1280
|
function AvatarVideoSurface(props) {
|
|
1206
1281
|
const {
|
|
@@ -1409,6 +1484,7 @@ function isFrameFreezeInhibited(snapshot) {
|
|
|
1409
1484
|
function useLiveFrameFlow(wrapRef, trackProducing, trackIdentity, stallAfterMs) {
|
|
1410
1485
|
const boundedStallMs = normalizeFrameStallMs(stallAfterMs);
|
|
1411
1486
|
const recovery = useMemo(() => new FrameRecovery(boundedStallMs), [trackIdentity, boundedStallMs]);
|
|
1487
|
+
const escalation = useMemo(() => new StallEscalation(boundedStallMs), [trackIdentity, boundedStallMs]);
|
|
1412
1488
|
const [flowing, setFlowing] = useState(false);
|
|
1413
1489
|
const [seenFrame, setSeenFrame] = useState(false);
|
|
1414
1490
|
const flowingRef = useRef(false);
|
|
@@ -1416,14 +1492,16 @@ function useLiveFrameFlow(wrapRef, trackProducing, trackIdentity, stallAfterMs)
|
|
|
1416
1492
|
seenFrame: false,
|
|
1417
1493
|
lastFrameAtMs: null,
|
|
1418
1494
|
maxGapMs: 0,
|
|
1419
|
-
resumePending: false
|
|
1495
|
+
resumePending: false,
|
|
1496
|
+
producingSinceMs: null
|
|
1420
1497
|
});
|
|
1421
1498
|
useEffect(() => {
|
|
1422
1499
|
sampleRef.current = {
|
|
1423
1500
|
seenFrame: false,
|
|
1424
1501
|
lastFrameAtMs: null,
|
|
1425
1502
|
maxGapMs: 0,
|
|
1426
|
-
resumePending: document.visibilityState !== "visible"
|
|
1503
|
+
resumePending: document.visibilityState !== "visible",
|
|
1504
|
+
producingSinceMs: trackProducing ? Date.now() : null
|
|
1427
1505
|
};
|
|
1428
1506
|
flowingRef.current = false;
|
|
1429
1507
|
setSeenFrame(false);
|
|
@@ -1441,6 +1519,7 @@ function useLiveFrameFlow(wrapRef, trackProducing, trackIdentity, stallAfterMs)
|
|
|
1441
1519
|
lastCurrentTime = video?.currentTime ?? lastCurrentTime;
|
|
1442
1520
|
const firstFrame = !previous.seenFrame;
|
|
1443
1521
|
sampleRef.current = {
|
|
1522
|
+
producingSinceMs: previous.producingSinceMs,
|
|
1444
1523
|
seenFrame: true,
|
|
1445
1524
|
lastFrameAtMs: now,
|
|
1446
1525
|
// A hidden tab/bfcache resume is a local scheduling gap, not network
|
|
@@ -1452,6 +1531,7 @@ function useLiveFrameFlow(wrapRef, trackProducing, trackIdentity, stallAfterMs)
|
|
|
1452
1531
|
resumePending: false
|
|
1453
1532
|
};
|
|
1454
1533
|
if (firstFrame) setSeenFrame(true);
|
|
1534
|
+
recovery.stallAfterMs = escalation.thresholdMs(now);
|
|
1455
1535
|
const nextFlowing = recovery.frame(now);
|
|
1456
1536
|
if (flowingRef.current !== nextFlowing) {
|
|
1457
1537
|
flowingRef.current = nextFlowing;
|
|
@@ -1497,13 +1577,19 @@ function useLiveFrameFlow(wrapRef, trackProducing, trackIdentity, stallAfterMs)
|
|
|
1497
1577
|
lastCurrentTime = video.currentTime;
|
|
1498
1578
|
markFrame();
|
|
1499
1579
|
}
|
|
1580
|
+
const nowMs = Date.now();
|
|
1581
|
+
const stallAfterMs2 = escalation.thresholdMs(nowMs);
|
|
1582
|
+
recovery.stallAfterMs = stallAfterMs2;
|
|
1500
1583
|
const frameFresh = isFrameFlowingAt({
|
|
1501
1584
|
seenFrame: sampleRef.current.seenFrame,
|
|
1502
1585
|
lastFrameAtMs: sampleRef.current.lastFrameAtMs,
|
|
1503
|
-
nowMs
|
|
1504
|
-
stallAfterMs:
|
|
1586
|
+
nowMs,
|
|
1587
|
+
stallAfterMs: stallAfterMs2
|
|
1505
1588
|
});
|
|
1506
|
-
if (!frameFresh && sampleRef.current.seenFrame)
|
|
1589
|
+
if (!frameFresh && sampleRef.current.seenFrame) {
|
|
1590
|
+
if (flowingRef.current) escalation.recordStall(nowMs);
|
|
1591
|
+
recovery.stall();
|
|
1592
|
+
}
|
|
1507
1593
|
const nextFlowing = frameFresh && recovery.ready;
|
|
1508
1594
|
if (flowingRef.current !== nextFlowing) {
|
|
1509
1595
|
flowingRef.current = nextFlowing;
|
|
@@ -1517,11 +1603,14 @@ function useLiveFrameFlow(wrapRef, trackProducing, trackIdentity, stallAfterMs)
|
|
|
1517
1603
|
stopObserving();
|
|
1518
1604
|
cancelFrameCallback();
|
|
1519
1605
|
};
|
|
1520
|
-
}, [wrapRef, trackProducing, trackIdentity, boundedStallMs, recovery]);
|
|
1606
|
+
}, [wrapRef, trackProducing, trackIdentity, boundedStallMs, recovery, escalation]);
|
|
1521
1607
|
const freezeReading = useCallback(() => {
|
|
1522
1608
|
const sample = sampleRef.current;
|
|
1523
1609
|
const lastFrameAtMs = sample.lastFrameAtMs;
|
|
1524
1610
|
const hidden = typeof document !== "undefined" && document.visibilityState !== "visible";
|
|
1611
|
+
if (!hidden && !sample.resumePending && trackProducing && !sample.seenFrame && sample.producingSinceMs !== null) {
|
|
1612
|
+
return { freezeMsInWindow: firstFrameWaitFreezeMs(Date.now() - sample.producingSinceMs), inhibited: false };
|
|
1613
|
+
}
|
|
1525
1614
|
const inhibited = isFrameFreezeInhibited({
|
|
1526
1615
|
hidden,
|
|
1527
1616
|
resumePending: sample.resumePending,
|
|
@@ -1771,8 +1860,7 @@ var sessionChoreographySchema = z.object({
|
|
|
1771
1860
|
{ message: "idle_dwell_min_seconds must be <= idle_dwell_max_seconds" }
|
|
1772
1861
|
);
|
|
1773
1862
|
var sessionBehaviorSchema = z.object({
|
|
1774
|
-
gestures_enabled: z.boolean().optional()
|
|
1775
|
-
gesture_freq: z.enum(["sparse", "balanced", "lively"]).optional()
|
|
1863
|
+
gestures_enabled: z.boolean().optional()
|
|
1776
1864
|
}).strict();
|
|
1777
1865
|
var transcriptWebhookSchema = z.object({
|
|
1778
1866
|
url: z.string().url().max(500),
|
|
@@ -3129,4 +3217,4 @@ function createProxyClient(options) {
|
|
|
3129
3217
|
};
|
|
3130
3218
|
}
|
|
3131
3219
|
|
|
3132
|
-
export { AdaptivePlayoutController, AvatarCall, AvatarVideoSurface, DEFAULT_APPROACHING_END_LEAD_SECONDS, DEFAULT_AVATAR_PLAYOUT_DELAY_SECONDS, DEFAULT_CREDITS_LOW_LEAD_SECONDS, DEFAULT_GOVERNOR_CONFIG, DEFAULT_GRACE_CEILING_SECONDS, DEFAULT_GRACE_WINDOW_LEAD_SECONDS, DEFAULT_IDLE_SECONDS, DEFAULT_IDLE_WARN_LEAD_SECONDS, DEFAULT_TURN_TIMEOUT_SECONDS, MAX_RECONNECT_ATTEMPTS, MAX_SESSION_INSTRUCTIONS_CHARS, MIC_LEASE_ENDED_TIMEOUT_MS, RECONNECT_BACKOFF_MS, RealtimeAvatarCapacityError, RealtimeAvatarLiveKitRoom, SessionLifecycleRoomBridge, capacityErrorFromBusy, capacityStateFromGrant, createProxyClient, knownBehaviorStates, mapTurnState, readInboundRtp, sessionBehaviorSchema, sessionClipSchema, shouldReplayPendingTurn, splitCallTranscript, useAvatarAdaptivePlayoutDelay, useAvatarCall, useAvatarPlayoutDelay, useAvatarQualityGovernor, useCallTranscript, useCharacterTools, useLiveKitAvatarGrant, useMicLease, useRealtimeSession, useReleaseMicLeaseOnTrackEnded, useSessionLifecycle };
|
|
3220
|
+
export { AdaptivePlayoutController, AvatarCall, AvatarVideoSurface, DEFAULT_APPROACHING_END_LEAD_SECONDS, DEFAULT_AVATAR_PLAYOUT_DELAY_SECONDS, DEFAULT_AVATAR_UNSTABLE_STALL_MS, DEFAULT_CREDITS_LOW_LEAD_SECONDS, DEFAULT_GOVERNOR_CONFIG, DEFAULT_GRACE_CEILING_SECONDS, DEFAULT_GRACE_WINDOW_LEAD_SECONDS, DEFAULT_IDLE_SECONDS, DEFAULT_IDLE_WARN_LEAD_SECONDS, DEFAULT_TURN_TIMEOUT_SECONDS, MAX_RECONNECT_ATTEMPTS, MAX_SESSION_INSTRUCTIONS_CHARS, MIC_LEASE_ENDED_TIMEOUT_MS, RECONNECT_BACKOFF_MS, RealtimeAvatarCapacityError, RealtimeAvatarLiveKitRoom, SessionLifecycleRoomBridge, capacityErrorFromBusy, capacityStateFromGrant, createProxyClient, knownBehaviorStates, mapTurnState, readInboundRtp, sessionBehaviorSchema, sessionClipSchema, shouldReplayPendingTurn, splitCallTranscript, useAvatarAdaptivePlayoutDelay, useAvatarCall, useAvatarPlayoutDelay, useAvatarQualityGovernor, useCallTranscript, useCharacterTools, useLiveKitAvatarGrant, useMicLease, useRealtimeSession, useReleaseMicLeaseOnTrackEnded, useSessionLifecycle };
|
package/dist/server.d.ts
CHANGED
|
@@ -1,3 +1,3 @@
|
|
|
1
1
|
export { RealtimeAvatar, RealtimeAvatarError, RealtimeAvatarHttpError, RealtimeAvatarOptions, StartCallOptions, clipLibraryDeclarationSchema, verifyTranscript } from './index.js';
|
|
2
|
-
export { h as Asset, g as AssetKind, A as Avatar, l as AvatarClip, c as AvatarSourceSwap, b as AvatarUpdate, m as CallConnection, C as CallMode, a as CallPolicy, n as CallQueued, o as ClipDeclaration, f as ClipLibrary, d as ClipLibraryDeclaration, p as ClipLibraryPlan, e as ClipLibraryUpdate, q as ClipSource, r as ContextMessage, k as CreditBalance, E as EndCallOptions, s as EndCallReason, i as ListSessionsOptions, L as LoopRedirect, S as StartCallResult, T as TranscriptPayload, j as UsageSession, U as UsageSessionPage, V as VideoPolicy, t as isQueued } from './types-
|
|
2
|
+
export { h as Asset, g as AssetKind, A as Avatar, l as AvatarClip, c as AvatarSourceSwap, b as AvatarUpdate, m as CallConnection, C as CallMode, a as CallPolicy, n as CallQueued, o as ClipDeclaration, f as ClipLibrary, d as ClipLibraryDeclaration, p as ClipLibraryPlan, e as ClipLibraryUpdate, q as ClipSource, r as ContextMessage, k as CreditBalance, E as EndCallOptions, s as EndCallReason, i as ListSessionsOptions, L as LoopRedirect, S as StartCallResult, T as TranscriptPayload, j as UsageSession, U as UsageSessionPage, V as VideoPolicy, t as isQueued } from './types-nxdohaiH.js';
|
|
3
3
|
import 'zod';
|
package/dist/server.js
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
export { RealtimeAvatar, RealtimeAvatarError, RealtimeAvatarHttpError, clipLibraryDeclarationSchema, isQueued, verifyTranscript } from './chunk-
|
|
1
|
+
export { RealtimeAvatar, RealtimeAvatarError, RealtimeAvatarHttpError, clipLibraryDeclarationSchema, isQueued, verifyTranscript } from './chunk-X3XXLAJJ.js';
|
package/dist/tanstack-start.d.ts
CHANGED
|
@@ -1,5 +1,5 @@
|
|
|
1
|
-
import { P as ProxyConfig } from './types-
|
|
2
|
-
import './types-
|
|
1
|
+
import { P as ProxyConfig } from './types-9a0bRdXl.js';
|
|
2
|
+
import './types-nxdohaiH.js';
|
|
3
3
|
|
|
4
4
|
/**
|
|
5
5
|
* TanStack Start. Mount at `routes/api/realtime-avatar/$.ts` — the trailing `$` is Start's
|
package/dist/tanstack-start.js
CHANGED
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
import { C as CallMode, a as CallPolicy } from './types-
|
|
1
|
+
import { C as CallMode, a as CallPolicy } from './types-nxdohaiH.js';
|
|
2
2
|
|
|
3
3
|
/** The three things a caller can ask the proxy to do. Gate on these, not on URLs. */
|
|
4
4
|
type ProxyOperation = "connect" | "end" | "avatars" | "credits";
|
|
@@ -132,8 +132,6 @@ interface components {
|
|
|
132
132
|
};
|
|
133
133
|
behavior?: {
|
|
134
134
|
gestures_enabled?: boolean;
|
|
135
|
-
/** @enum {string} */
|
|
136
|
-
gesture_freq?: "sparse" | "balanced" | "lively";
|
|
137
135
|
};
|
|
138
136
|
expression_profile?: string;
|
|
139
137
|
/** @enum {string} */
|
|
@@ -647,6 +645,11 @@ interface components {
|
|
|
647
645
|
documentation: string;
|
|
648
646
|
/** @description Set when the failure is worth retrying as-is. */
|
|
649
647
|
retryable?: boolean;
|
|
648
|
+
/**
|
|
649
|
+
* Format: uuid
|
|
650
|
+
* @description Server-generated correlation ID, also returned in X-Request-ID on correlated failures. Quote this when contacting support. Detailed diagnostics require separate operational access.
|
|
651
|
+
*/
|
|
652
|
+
requestId?: string;
|
|
650
653
|
/**
|
|
651
654
|
* @description Present on `403 insufficient_scope` — the one scope this operation needed and the key did not carry. Repeated in the `WWW-Authenticate` challenge as `scope=`.
|
|
652
655
|
* @enum {string}
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "realtime-avatar",
|
|
3
|
-
"version": "0.
|
|
3
|
+
"version": "0.11.1",
|
|
4
4
|
"description": "Realtime Avatar SDK. `realtime-avatar` and the route adapters hold your API key and are server-only; `realtime-avatar/react`, `/browser` and `/tools` never can — importing a server entry into a browser build throws at bundle time.",
|
|
5
5
|
"license": "MIT",
|
|
6
6
|
"author": "The Influence Company",
|