realtime-avatar 0.12.0 → 0.14.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +20 -1
- package/dist/{chunk-FQWEKJCX.js → chunk-B3AHTJ3J.js} +1 -1
- package/dist/{chunk-ZFYNUHKB.js → chunk-UZQXZH2Z.js} +1 -1
- package/dist/express.js +2 -2
- package/dist/hono.js +2 -2
- package/dist/index.js +1 -1
- package/dist/nextjs.js +2 -2
- package/dist/{proxy-client-8cQOLLE_.d.ts → proxy-client-DhVdbfQX.d.ts} +34 -12
- package/dist/react-native.d.ts +2 -2
- package/dist/react-native.js +165 -28
- package/dist/react.d.ts +2 -2
- package/dist/react.js +168 -29
- package/dist/server.js +1 -1
- package/dist/tanstack-start.js +2 -2
- package/package.json +1 -1
package/README.md
CHANGED
|
@@ -4,7 +4,7 @@ A live character your users can talk to — voice, or voice and video. She liste
|
|
|
4
4
|
speaks, so you can interrupt her mid-sentence and she stops, the way a person stops.
|
|
5
5
|
|
|
6
6
|
```bash
|
|
7
|
-
npm install realtime-avatar
|
|
7
|
+
npm install --save-exact realtime-avatar@0.14.0
|
|
8
8
|
```
|
|
9
9
|
|
|
10
10
|
```ts
|
|
@@ -147,6 +147,25 @@ call. Policy — `instructions`, `maxSeconds`, `voice`, `video` — is decided i
|
|
|
147
147
|
your server. A route that spreads the request body into `startCall` hands your caller your
|
|
148
148
|
system prompt and your bill.
|
|
149
149
|
|
|
150
|
+
### Native connection facts
|
|
151
|
+
|
|
152
|
+
LiveKit owns media transport, connection state, and reconnection. Use its
|
|
153
|
+
`useConnectionState()` and `useConnectionQualityIndicator({ participant })` hooks, or
|
|
154
|
+
`RoomEvent` callbacks, to build your product's network UI. Their values already use
|
|
155
|
+
`ConnectionState`, `ConnectionQuality`, and `Track.StreamState` from `livekit-client`.
|
|
156
|
+
For current receiver measurements, `RemoteVideoTrack.getReceiverStats()` and
|
|
157
|
+
`RemoteAudioTrack.getReceiverStats()` return LiveKit's exported `VideoReceiverStats` and
|
|
158
|
+
`AudioReceiverStats` types.
|
|
159
|
+
Keep these native types in process; there is no SDK-specific JSON mirror to maintain.
|
|
160
|
+
|
|
161
|
+
Retain the app's session-to-room association (`room_name`, timestamps and participant
|
|
162
|
+
identity), adding the server-observed room SID when exact room-lifetime lookup needs it.
|
|
163
|
+
LiveKit owns participant connection history; a participant SID identifies one incarnation.
|
|
164
|
+
Use these references to look up details in LiveKit's own diagnostics. The existing quality governor
|
|
165
|
+
and opt-in adaptive playout are product policies over LiveKit facts; setting
|
|
166
|
+
`adaptiveQuality={false}` releases the manual quality ceiling while LiveKit's bandwidth
|
|
167
|
+
adaptation continues. Adaptive playout reads only public LiveKit track reports.
|
|
168
|
+
|
|
150
169
|
### Importing a server entry into a browser build throws
|
|
151
170
|
|
|
152
171
|
Not a lint rule and not a naming convention — the six server subpaths carry `browser` and
|
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
import { RealtimeAvatar, isQueued, RealtimeAvatarHttpError } from './chunk-
|
|
1
|
+
import { RealtimeAvatar, isQueued, RealtimeAvatarHttpError } from './chunk-UZQXZH2Z.js';
|
|
2
2
|
|
|
3
3
|
// ../proxy/src/config.ts
|
|
4
4
|
var json = (body, status = 200) => new Response(JSON.stringify(body), {
|
|
@@ -236,7 +236,7 @@ function isTransient(cause) {
|
|
|
236
236
|
|
|
237
237
|
// ../http-client/src/client.ts
|
|
238
238
|
var DEFAULT_BASE_URL = "https://realtimeavatar.ai/api/v1";
|
|
239
|
-
var SDK_VERSION = "0.
|
|
239
|
+
var SDK_VERSION = "0.14.0";
|
|
240
240
|
var RealtimeAvatar = class {
|
|
241
241
|
#apiKey;
|
|
242
242
|
#baseUrl;
|
package/dist/express.js
CHANGED
package/dist/hono.js
CHANGED
package/dist/index.js
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
export { RealtimeAvatar, RealtimeAvatarError, RealtimeAvatarHttpError, clipLibraryDeclarationSchema, isQueued, verifyTranscript } from './chunk-
|
|
1
|
+
export { RealtimeAvatar, RealtimeAvatarError, RealtimeAvatarHttpError, clipLibraryDeclarationSchema, isQueued, verifyTranscript } from './chunk-UZQXZH2Z.js';
|
package/dist/nextjs.js
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
import { TrackReferenceOrPlaceholder, useTranscriptions, useConnectionState, LiveKitRoomProps } from '@livekit/components-react';
|
|
2
2
|
import { CSSProperties, ReactNode, ReactElement } from 'react';
|
|
3
|
-
import { DisconnectReason, SendTextOptions } from 'livekit-client';
|
|
3
|
+
import { ConnectionQuality, DisconnectReason, SendTextOptions } from 'livekit-client';
|
|
4
4
|
import { z } from 'zod';
|
|
5
5
|
|
|
6
6
|
/**
|
|
@@ -312,7 +312,7 @@ type GovernorState = "opening" | "opening_high" | "cap_low_sticky" | "cap_low_el
|
|
|
312
312
|
/**
|
|
313
313
|
* One normalized input the hook feeds each tick. The hook translates ALL of
|
|
314
314
|
* {TrackStreamStateChanged, ConnectionQualityChanged, getStats, rVFC freeze} into
|
|
315
|
-
* this shape
|
|
315
|
+
* this shape. LiveKit owns the connection-quality vocabulary.
|
|
316
316
|
*/
|
|
317
317
|
interface GovernorSignal {
|
|
318
318
|
/** SFU congestion controller paused the track since the last tick — Tier-0, the
|
|
@@ -323,9 +323,8 @@ interface GovernorSignal {
|
|
|
323
323
|
freezeMsInWindow: number;
|
|
324
324
|
/** jitterBufferDelay is trending up — the earliest LEADING pre-freeze sign. */
|
|
325
325
|
jitterRising: boolean;
|
|
326
|
-
/** LiveKit
|
|
327
|
-
|
|
328
|
-
connectionQuality: "excellent" | "good" | "poor" | "lost" | "unknown";
|
|
326
|
+
/** LiveKit connection quality blocks up-probes; it is never a sole downgrade trigger. */
|
|
327
|
+
connectionQuality: `${ConnectionQuality}`;
|
|
329
328
|
/** The tab is hidden / track muted / freezes correlate with local CPU not network —
|
|
330
329
|
* the false-positive fence (our own Dia-freeze lesson). When true the machine is
|
|
331
330
|
* frozen: no signal is trusted, no transition fires. */
|
|
@@ -403,6 +402,22 @@ interface Governor {
|
|
|
403
402
|
healthySinceMs: number | null;
|
|
404
403
|
}
|
|
405
404
|
|
|
405
|
+
declare const NETWORK_QUALITY_POLICY: {
|
|
406
|
+
readonly startupGraceMs: 3000;
|
|
407
|
+
readonly maxSampleGapMs: 1500;
|
|
408
|
+
readonly minimumPackets: 20;
|
|
409
|
+
readonly lossWindowSamples: 3;
|
|
410
|
+
readonly lossRatio: 0.05;
|
|
411
|
+
readonly jitterSeconds: 0.1;
|
|
412
|
+
readonly impairedFps: 12;
|
|
413
|
+
readonly downgradeMs: 2000;
|
|
414
|
+
readonly floorMs: 2000;
|
|
415
|
+
readonly notifyAfterDowngradeMs: 5000;
|
|
416
|
+
readonly recoverMs: 5000;
|
|
417
|
+
readonly clearNoticeMs: 10000;
|
|
418
|
+
};
|
|
419
|
+
type NetworkQualityStatus = "unknown" | "healthy" | "poor";
|
|
420
|
+
|
|
406
421
|
/** How the media is fit into the surface box. Mirrors CSS `object-fit`. */
|
|
407
422
|
type AvatarVideoFit = "contain" | "cover";
|
|
408
423
|
type AvatarVideoSurfaceProps = {
|
|
@@ -439,8 +454,13 @@ type AvatarVideoSurfaceProps = {
|
|
|
439
454
|
* Enable the subscriber-side fast-down/sticky-up quality governor. It starts on
|
|
440
455
|
* the lower simulcast rung, reacts immediately to an SFU pause or decoded-frame
|
|
441
456
|
* freeze, and only probes the full layer after a clean dwell. Default true.
|
|
457
|
+
* False releases the manual cap to HIGH on each subscription without running
|
|
458
|
+
* the governor. SFU bandwidth adaptation and live/idle fallback remain active.
|
|
442
459
|
*/
|
|
443
|
-
adaptiveQuality?: boolean;
|
|
460
|
+
adaptiveQuality?: boolean | "network-only";
|
|
461
|
+
/** Opt-in "network-only" keeps HIGH unless sustained receiver/network impairment is
|
|
462
|
+
* observed. Status is unknown until enough fresh data exists; poor after 5s. */
|
|
463
|
+
onNetworkStatusChange?: (status: NetworkQualityStatus) => void;
|
|
444
464
|
/**
|
|
445
465
|
* Opening bet for the governor when `adaptiveQuality` is on. `"high"` opens on the
|
|
446
466
|
* FULL simulcast layer under probation — sharp from frame 1, no LOW→HIGH layer-walk —
|
|
@@ -966,7 +986,7 @@ declare function readInboundRtp(reports: Iterable<Record<string, unknown>>, prev
|
|
|
966
986
|
* (default false ⇒ a complete no-op, the incumbent flat cushion untouched).
|
|
967
987
|
*
|
|
968
988
|
* When enabled it runs the {@link AdaptivePlayoutController} closed loop: a 1Hz
|
|
969
|
-
* `
|
|
989
|
+
* LiveKit `getRTCStatsReport()` poll over BOTH avatar tracks, the worst
|
|
970
990
|
* jitter/loss of the pair feeding one shared target so audio and video stay on
|
|
971
991
|
* the SAME depth (lip-sync law — WebRTC pairs the streams to the larger of the
|
|
972
992
|
* two hints). The loop opens at the ceiling (the flat cushion already applied by
|
|
@@ -974,7 +994,7 @@ declare function readInboundRtp(reports: Iterable<Record<string, unknown>>, prev
|
|
|
974
994
|
* snaps back up within a couple of ticks when jitter or loss appears.
|
|
975
995
|
*
|
|
976
996
|
* Degrades to the incumbent behavior wherever the substrate is missing: a track
|
|
977
|
-
* without
|
|
997
|
+
* without an available LiveKit stats report, a report with no `inbound-rtp` yet,
|
|
978
998
|
* or a browser without the playout-delay hint all leave the flat cushion exactly
|
|
979
999
|
* as `useAvatarPlayoutDelay` set it.
|
|
980
1000
|
*
|
|
@@ -1539,9 +1559,11 @@ type FreezeReadingFn = () => {
|
|
|
1539
1559
|
inhibited: boolean;
|
|
1540
1560
|
};
|
|
1541
1561
|
interface UseAvatarQualityGovernorInput {
|
|
1542
|
-
/** Master switch (product policy
|
|
1543
|
-
*
|
|
1544
|
-
enabled: boolean;
|
|
1562
|
+
/** Master switch (product policy). Off releases the manual cap to HIGH on each
|
|
1563
|
+
* subscription, with no governor tick or congestion listeners. SFU adaptation stays on. */
|
|
1564
|
+
enabled: boolean | "network-only";
|
|
1565
|
+
/** Sustained, receiver-evidenced impairment. Never inferred from rendering jitter alone. */
|
|
1566
|
+
onNetworkStatusChange?: (status: NetworkQualityStatus) => void;
|
|
1545
1567
|
/** The player's rVFC freeze reading getter (see FreezeReadingFn). Optional: without
|
|
1546
1568
|
* it the governor still reacts to Paused + getStats freezes, just without the
|
|
1547
1569
|
* cross-browser rVFC signal. */
|
|
@@ -1624,4 +1646,4 @@ interface ProxyClientOptions {
|
|
|
1624
1646
|
}
|
|
1625
1647
|
declare function createProxyClient(options: ProxyClientOptions): AvatarSessionClient;
|
|
1626
1648
|
|
|
1627
|
-
export { RECONNECT_BACKOFF_MS as $, type AvatarVideoFit as A, type BehaviorSnapshot as B, type CallTranscript as C, DEFAULT_APPROACHING_END_LEAD_SECONDS as D, type EndReason as E, type FishTtsModel as F, type Governor as G, type GraceWindowClosedEvent as H, type GraceWindowOpenEvent as I, type GraceWindowState as J, type IdleWarningEvent as K, type LiveKitSessionGrant as L, type InboundRtpCursor as M, type InboundRtpReading as N, type KnownBehaviorState as O, type LLMProvider as P, type LLMSelection as Q, type LiveKitAvatarGrantState as R, type LiveKitAvatarGrantStatus as S, type LiveKitCapacityState as T, type LiveKitConnectionStatus as U, type LiveKitSessionRequest as V, type LiveKitSessionStartResult as W, MAX_RECONNECT_ATTEMPTS as X, MAX_SESSION_INSTRUCTIONS_CHARS as Y, type ProxyClientOptions as Z, type QualityCap as _, AdaptivePlayoutController as a, type RealtimeAvatarRequestOptions as a0, type RealtimeSessionApi as a1, type RealtimeSessionMedia as a2, type RealtimeSessionRoomSinks as a3, type ReconnectPolicy as a4, type ReconnectingEvent as a5, type RecoveryState as a6, type RetryStep as a7, type SessionBehavior as a8, type SessionClip as a9, useAvatarAdaptivePlayoutDelay as aA, useAvatarPlayoutDelay as aB, useAvatarQualityGovernor as aC, useCallTranscript as aD, useLiveKitAvatarGrant as aE, useLiveTrackProducing as aF, useMicLease as aG, useRealtimeSession as aH, useReleaseMicLeaseOnTrackEnded as aI, useSessionLifecycle as aJ, AvatarVideoSurface as aK, type AvatarVideoSurfaceProps as aL, type LivePlaybackKeeper as aM, MIC_LEASE_ENDED_TIMEOUT_MS as aN, type MicLease as aO, type PlayableVideoElement as
|
|
1649
|
+
export { RECONNECT_BACKOFF_MS as $, type AvatarVideoFit as A, type BehaviorSnapshot as B, type CallTranscript as C, DEFAULT_APPROACHING_END_LEAD_SECONDS as D, type EndReason as E, type FishTtsModel as F, type Governor as G, type GraceWindowClosedEvent as H, type GraceWindowOpenEvent as I, type GraceWindowState as J, type IdleWarningEvent as K, type LiveKitSessionGrant as L, type InboundRtpCursor as M, type InboundRtpReading as N, type KnownBehaviorState as O, type LLMProvider as P, type LLMSelection as Q, type LiveKitAvatarGrantState as R, type LiveKitAvatarGrantStatus as S, type LiveKitCapacityState as T, type LiveKitConnectionStatus as U, type LiveKitSessionRequest as V, type LiveKitSessionStartResult as W, MAX_RECONNECT_ATTEMPTS as X, MAX_SESSION_INSTRUCTIONS_CHARS as Y, type ProxyClientOptions as Z, type QualityCap as _, AdaptivePlayoutController as a, type RealtimeAvatarRequestOptions as a0, type RealtimeSessionApi as a1, type RealtimeSessionMedia as a2, type RealtimeSessionRoomSinks as a3, type ReconnectPolicy as a4, type ReconnectingEvent as a5, type RecoveryState as a6, type RetryStep as a7, type SessionBehavior as a8, type SessionClip as a9, useAvatarAdaptivePlayoutDelay as aA, useAvatarPlayoutDelay as aB, useAvatarQualityGovernor as aC, useCallTranscript as aD, useLiveKitAvatarGrant as aE, useLiveTrackProducing as aF, useMicLease as aG, useRealtimeSession as aH, useReleaseMicLeaseOnTrackEnded as aI, useSessionLifecycle as aJ, AvatarVideoSurface as aK, type AvatarVideoSurfaceProps as aL, type LivePlaybackKeeper as aM, MIC_LEASE_ENDED_TIMEOUT_MS as aN, type MicLease as aO, NETWORK_QUALITY_POLICY as aP, type NetworkQualityStatus as aQ, type PlayableVideoElement as aR, RealtimeAvatarCapacityError as aS, RealtimeAvatarLiveKitRoom as aT, type RealtimeAvatarLiveKitRoomProps as aU, shouldReplayPendingTurn as aV, type SessionClocks as aa, type SessionEndReason as ab, type SessionLifecycleApi as ac, type SessionLifecyclePhase as ad, type SessionLifecyclePhaseKind as ae, SessionLifecycleRoomBridge as af, type SessionLifecycleRoomBridgeProps as ag, type SurfaceLayers as ah, type TurnState as ai, type TurnTimeoutEvent as aj, type UseAvatarQualityGovernorInput as ak, type UseLiveKitAvatarGrantInput as al, type UseRealtimeSessionInput as am, type UseSessionLifecycleInput as an, type VoiceSpec as ao, type VoiceSpecInput as ap, capacityErrorFromBusy as aq, capacityStateFromGrant as ar, createProxyClient as as, isNativeLiveTrackSubscribed as at, knownBehaviorStates as au, mapTurnState as av, readInboundRtp as aw, sessionBehaviorSchema as ax, sessionClipSchema as ay, splitCallTranscript as az, type AdaptivePlayoutDecision as b, type AdaptivePlayoutOptions as c, type AdaptivePlayoutSample as d, type ApproachingEndEvent as e, type ApproachingEndReason as f, type AvatarSessionClient as g, type CallTranscriptSegment as h, type CartesiaTtsModel as i, type ClipResult as j, type ClosingTurnResult as k, type CreditsLowEvent as l, DEFAULT_CREDITS_LOW_LEAD_SECONDS as m, DEFAULT_GOVERNOR_CONFIG as n, DEFAULT_GRACE_CEILING_SECONDS as o, DEFAULT_GRACE_WINDOW_LEAD_SECONDS as p, DEFAULT_IDLE_SECONDS as q, DEFAULT_IDLE_WARN_LEAD_SECONDS as r, DEFAULT_TURN_TIMEOUT_SECONDS as s, type EndedEvent as t, type ExtendResult as u, type FreezeReadingFn as v, type GovernorAction as w, type GovernorConfig as x, type GovernorSignal as y, type GovernorState as z };
|
package/dist/react-native.d.ts
CHANGED
|
@@ -1,8 +1,8 @@
|
|
|
1
1
|
import { LiveKitRoomProps } from '@livekit/react-native';
|
|
2
2
|
export { AudioSession, VideoTrack, VideoTrackProps, registerGlobals } from '@livekit/react-native';
|
|
3
3
|
import { ReactNode, ReactElement } from 'react';
|
|
4
|
-
import { L as LiveKitSessionGrant, A as AvatarVideoFit } from './proxy-client-
|
|
5
|
-
export { a as AdaptivePlayoutController, b as AdaptivePlayoutDecision, c as AdaptivePlayoutOptions, d as AdaptivePlayoutSample, e as ApproachingEndEvent, f as ApproachingEndReason, g as AvatarSessionClient, B as BehaviorSnapshot, C as CallTranscript, h as CallTranscriptSegment, i as CartesiaTtsModel, j as ClipResult, k as ClosingTurnResult, l as CreditsLowEvent, D as DEFAULT_APPROACHING_END_LEAD_SECONDS, m as DEFAULT_CREDITS_LOW_LEAD_SECONDS, n as DEFAULT_GOVERNOR_CONFIG, o as DEFAULT_GRACE_CEILING_SECONDS, p as DEFAULT_GRACE_WINDOW_LEAD_SECONDS, q as DEFAULT_IDLE_SECONDS, r as DEFAULT_IDLE_WARN_LEAD_SECONDS, s as DEFAULT_TURN_TIMEOUT_SECONDS, E as EndReason, t as EndedEvent, u as ExtendResult, F as FishTtsModel, v as FreezeReadingFn, G as Governor, w as GovernorAction, x as GovernorConfig, y as GovernorSignal, z as GovernorState, H as GraceWindowClosedEvent, I as GraceWindowOpenEvent, J as GraceWindowState, K as IdleWarningEvent, M as InboundRtpCursor, N as InboundRtpReading, O as KnownBehaviorState, P as LLMProvider, Q as LLMSelection, R as LiveKitAvatarGrantState, S as LiveKitAvatarGrantStatus, T as LiveKitCapacityState, U as LiveKitConnectionStatus, V as LiveKitSessionRequest, W as LiveKitSessionStartResult, X as MAX_RECONNECT_ATTEMPTS, Y as MAX_SESSION_INSTRUCTIONS_CHARS, Z as ProxyClientOptions, _ as QualityCap, $ as RECONNECT_BACKOFF_MS, a0 as RealtimeAvatarRequestOptions, a1 as RealtimeSessionApi, a2 as RealtimeSessionMedia, a3 as RealtimeSessionRoomSinks, a4 as ReconnectPolicy, a5 as ReconnectingEvent, a6 as RecoveryState, a7 as RetryStep, a8 as SessionBehavior, a9 as SessionClip, aa as SessionClocks, ab as SessionEndReason, ac as SessionLifecycleApi, ad as SessionLifecyclePhase, ae as SessionLifecyclePhaseKind, af as SessionLifecycleRoomBridge, ag as SessionLifecycleRoomBridgeProps, ah as SurfaceLayers, ai as TurnState, aj as TurnTimeoutEvent, ak as UseAvatarQualityGovernorInput, al as UseLiveKitAvatarGrantInput, am as UseRealtimeSessionInput, an as UseSessionLifecycleInput, ao as VoiceSpec, ap as VoiceSpecInput, aq as capacityErrorFromBusy, ar as capacityStateFromGrant, as as createProxyClient, at as isNativeLiveTrackSubscribed, au as knownBehaviorStates, av as mapTurnState, aw as readInboundRtp, ax as sessionBehaviorSchema, ay as sessionClipSchema, az as splitCallTranscript, aA as useAvatarAdaptivePlayoutDelay, aB as useAvatarPlayoutDelay, aC as useAvatarQualityGovernor, aD as useCallTranscript, aE as useLiveKitAvatarGrant, aF as useLiveTrackProducing, aG as useMicLease, aH as useRealtimeSession, aI as useReleaseMicLeaseOnTrackEnded, aJ as useSessionLifecycle } from './proxy-client-
|
|
4
|
+
import { L as LiveKitSessionGrant, A as AvatarVideoFit } from './proxy-client-DhVdbfQX.js';
|
|
5
|
+
export { a as AdaptivePlayoutController, b as AdaptivePlayoutDecision, c as AdaptivePlayoutOptions, d as AdaptivePlayoutSample, e as ApproachingEndEvent, f as ApproachingEndReason, g as AvatarSessionClient, B as BehaviorSnapshot, C as CallTranscript, h as CallTranscriptSegment, i as CartesiaTtsModel, j as ClipResult, k as ClosingTurnResult, l as CreditsLowEvent, D as DEFAULT_APPROACHING_END_LEAD_SECONDS, m as DEFAULT_CREDITS_LOW_LEAD_SECONDS, n as DEFAULT_GOVERNOR_CONFIG, o as DEFAULT_GRACE_CEILING_SECONDS, p as DEFAULT_GRACE_WINDOW_LEAD_SECONDS, q as DEFAULT_IDLE_SECONDS, r as DEFAULT_IDLE_WARN_LEAD_SECONDS, s as DEFAULT_TURN_TIMEOUT_SECONDS, E as EndReason, t as EndedEvent, u as ExtendResult, F as FishTtsModel, v as FreezeReadingFn, G as Governor, w as GovernorAction, x as GovernorConfig, y as GovernorSignal, z as GovernorState, H as GraceWindowClosedEvent, I as GraceWindowOpenEvent, J as GraceWindowState, K as IdleWarningEvent, M as InboundRtpCursor, N as InboundRtpReading, O as KnownBehaviorState, P as LLMProvider, Q as LLMSelection, R as LiveKitAvatarGrantState, S as LiveKitAvatarGrantStatus, T as LiveKitCapacityState, U as LiveKitConnectionStatus, V as LiveKitSessionRequest, W as LiveKitSessionStartResult, X as MAX_RECONNECT_ATTEMPTS, Y as MAX_SESSION_INSTRUCTIONS_CHARS, Z as ProxyClientOptions, _ as QualityCap, $ as RECONNECT_BACKOFF_MS, a0 as RealtimeAvatarRequestOptions, a1 as RealtimeSessionApi, a2 as RealtimeSessionMedia, a3 as RealtimeSessionRoomSinks, a4 as ReconnectPolicy, a5 as ReconnectingEvent, a6 as RecoveryState, a7 as RetryStep, a8 as SessionBehavior, a9 as SessionClip, aa as SessionClocks, ab as SessionEndReason, ac as SessionLifecycleApi, ad as SessionLifecyclePhase, ae as SessionLifecyclePhaseKind, af as SessionLifecycleRoomBridge, ag as SessionLifecycleRoomBridgeProps, ah as SurfaceLayers, ai as TurnState, aj as TurnTimeoutEvent, ak as UseAvatarQualityGovernorInput, al as UseLiveKitAvatarGrantInput, am as UseRealtimeSessionInput, an as UseSessionLifecycleInput, ao as VoiceSpec, ap as VoiceSpecInput, aq as capacityErrorFromBusy, ar as capacityStateFromGrant, as as createProxyClient, at as isNativeLiveTrackSubscribed, au as knownBehaviorStates, av as mapTurnState, aw as readInboundRtp, ax as sessionBehaviorSchema, ay as sessionClipSchema, az as splitCallTranscript, aA as useAvatarAdaptivePlayoutDelay, aB as useAvatarPlayoutDelay, aC as useAvatarQualityGovernor, aD as useCallTranscript, aE as useLiveKitAvatarGrant, aF as useLiveTrackProducing, aG as useMicLease, aH as useRealtimeSession, aI as useReleaseMicLeaseOnTrackEnded, aJ as useSessionLifecycle } from './proxy-client-DhVdbfQX.js';
|
|
6
6
|
import { StyleProp, ViewStyle } from 'react-native';
|
|
7
7
|
export { useChat, useConnectionState, useLocalParticipant, useRoomContext, useTrackToggle, useTranscriptions, useVoiceAssistant } from '@livekit/components-react';
|
|
8
8
|
export { D as DEFAULT_AVATAR_PLAYOUT_DELAY_SECONDS } from './playout-delay-DWOc8iCl.js';
|
package/dist/react-native.js
CHANGED
|
@@ -3,7 +3,7 @@ export { AudioSession, VideoTrack, registerGlobals } from '@livekit/react-native
|
|
|
3
3
|
import { useEffect, createElement, useRef, useState, useCallback, useMemo } from 'react';
|
|
4
4
|
import { useTranscriptions, useLocalParticipant, useMaybeRoomContext, useVoiceAssistant, useConnectionState, isTrackReference, useRoomContext, useChat } from '@livekit/components-react';
|
|
5
5
|
export { useChat, useConnectionState, useLocalParticipant, useRoomContext, useTrackToggle, useTranscriptions, useVoiceAssistant } from '@livekit/components-react';
|
|
6
|
-
import { ConnectionQuality, RoomEvent, TrackEvent, ConnectionState, Track,
|
|
6
|
+
import { isRemoteTrack, VideoQuality, ConnectionQuality, RoomEvent, TrackEvent, ConnectionState, Track, ConnectionError, ConnectionErrorReason, DisconnectReason, Room } from 'livekit-client';
|
|
7
7
|
export { DisconnectReason, Room, RoomEvent, Track } from 'livekit-client';
|
|
8
8
|
import { Platform, StyleSheet, Animated, Image, View, Text } from 'react-native';
|
|
9
9
|
import { z } from 'zod';
|
|
@@ -679,10 +679,8 @@ function useAvatarAdaptivePlayoutDelay(videoTrack, audioTrack, enabled = false,
|
|
|
679
679
|
const audioMediaTrack = audioTrack?.publication?.track;
|
|
680
680
|
useEffect(() => {
|
|
681
681
|
if (!enabled || !videoMediaTrack && !audioMediaTrack) return;
|
|
682
|
-
const
|
|
683
|
-
|
|
684
|
-
);
|
|
685
|
-
if (receivers.length === 0) return;
|
|
682
|
+
const tracks = [videoMediaTrack, audioMediaTrack].filter(isRemoteTrack);
|
|
683
|
+
if (tracks.length === 0) return;
|
|
686
684
|
const controller = new AdaptivePlayoutController(options);
|
|
687
685
|
const cursors = /* @__PURE__ */ new Map();
|
|
688
686
|
let closed = false;
|
|
@@ -692,9 +690,9 @@ function useAvatarAdaptivePlayoutDelay(videoTrack, audioTrack, enabled = false,
|
|
|
692
690
|
inFlight = true;
|
|
693
691
|
try {
|
|
694
692
|
let worst;
|
|
695
|
-
for (let i = 0; i <
|
|
696
|
-
const stats = await
|
|
697
|
-
if (closed ||
|
|
693
|
+
for (let i = 0; i < tracks.length; i++) {
|
|
694
|
+
const stats = await tracks[i].getRTCStatsReport();
|
|
695
|
+
if (closed || !stats) continue;
|
|
698
696
|
const reading = readInboundRtp(stats.values(), cursors.get(i));
|
|
699
697
|
if (!reading) continue;
|
|
700
698
|
cursors.set(i, reading.cursor);
|
|
@@ -877,23 +875,147 @@ var resolveLowCapQuality = (declaredLayerQualities, lowUnhealthy = 0) => {
|
|
|
877
875
|
return lowUnhealthy >= LOW_CAP_STEP_AFTER_UNHEALTHY ? sorted[0] : sorted[sorted.length - 2];
|
|
878
876
|
};
|
|
879
877
|
|
|
880
|
-
// ../client/src/react/
|
|
881
|
-
var
|
|
882
|
-
|
|
883
|
-
|
|
884
|
-
|
|
885
|
-
|
|
886
|
-
|
|
887
|
-
|
|
888
|
-
|
|
889
|
-
|
|
890
|
-
|
|
891
|
-
|
|
892
|
-
|
|
893
|
-
}
|
|
878
|
+
// ../client/src/react/network-quality.ts
|
|
879
|
+
var NETWORK_QUALITY_POLICY = {
|
|
880
|
+
startupGraceMs: 3e3,
|
|
881
|
+
maxSampleGapMs: 1500,
|
|
882
|
+
minimumPackets: 20,
|
|
883
|
+
lossRatio: 0.05,
|
|
884
|
+
jitterSeconds: 0.1,
|
|
885
|
+
impairedFps: 12,
|
|
886
|
+
downgradeMs: 2e3,
|
|
887
|
+
floorMs: 2e3,
|
|
888
|
+
notifyAfterDowngradeMs: 5e3,
|
|
889
|
+
recoverMs: 5e3,
|
|
890
|
+
clearNoticeMs: 1e4
|
|
894
891
|
};
|
|
892
|
+
function readNetworkVideoSample(report) {
|
|
893
|
+
const samples = [];
|
|
894
|
+
report?.forEach((row) => {
|
|
895
|
+
if (row.type !== "inbound-rtp" || (row.kind ?? row.mediaType) !== "video") return;
|
|
896
|
+
const { timestamp, framesDecoded, packetsReceived, packetsLost, jitter } = row;
|
|
897
|
+
if (typeof row.id !== "string" || ![timestamp, framesDecoded, packetsReceived, packetsLost, jitter].every(
|
|
898
|
+
(n) => typeof n === "number" && Number.isFinite(n)
|
|
899
|
+
) || timestamp < 0 || framesDecoded < 0 || packetsReceived < 0 || jitter < 0) return;
|
|
900
|
+
samples.push({
|
|
901
|
+
id: `${row.id}:${row.ssrc ?? ""}`,
|
|
902
|
+
timestamp,
|
|
903
|
+
framesDecoded,
|
|
904
|
+
packetsReceived,
|
|
905
|
+
packetsLost,
|
|
906
|
+
jitter
|
|
907
|
+
});
|
|
908
|
+
});
|
|
909
|
+
return samples.length === 1 ? samples[0] : null;
|
|
910
|
+
}
|
|
911
|
+
function initNetworkQuality(nowMs) {
|
|
912
|
+
return {
|
|
913
|
+
cap: "high",
|
|
914
|
+
status: "unknown",
|
|
915
|
+
sample: null,
|
|
916
|
+
lastPollMs: null,
|
|
917
|
+
openedMs: nowMs,
|
|
918
|
+
badMs: 0,
|
|
919
|
+
postDowngradeBadMs: 0,
|
|
920
|
+
reducedBadMs: 0,
|
|
921
|
+
cleanMs: 0,
|
|
922
|
+
packetWindow: []
|
|
923
|
+
};
|
|
924
|
+
}
|
|
925
|
+
function stepNetworkQuality(previous, sample, nowMs, options) {
|
|
926
|
+
const p = NETWORK_QUALITY_POLICY;
|
|
927
|
+
const s = { ...previous, sample, lastPollMs: nowMs };
|
|
928
|
+
const before = previous.sample;
|
|
929
|
+
const pollMs = previous.lastPollMs === null ? 0 : nowMs - previous.lastPollMs;
|
|
930
|
+
const elapsedMs = sample && before ? sample.timestamp - before.timestamp : 0;
|
|
931
|
+
const interrupted = options.inhibited || pollMs <= 0 || pollMs > p.maxSampleGapMs;
|
|
932
|
+
const valid = !interrupted && sample !== null && before !== null && sample.id === before.id && elapsedMs > 0 && elapsedMs <= p.maxSampleGapMs && sample.framesDecoded >= before.framesDecoded && sample.packetsReceived >= before.packetsReceived;
|
|
933
|
+
if (!valid || nowMs - s.openedMs < p.startupGraceMs) {
|
|
934
|
+
s.badMs = s.postDowngradeBadMs = s.reducedBadMs = s.cleanMs = 0;
|
|
935
|
+
s.packetWindow = [];
|
|
936
|
+
if (options.inhibited) s.sample = null;
|
|
937
|
+
return s;
|
|
938
|
+
}
|
|
939
|
+
const received = sample.packetsReceived - before.packetsReceived;
|
|
940
|
+
const lost = Math.max(0, sample.packetsLost - before.packetsLost);
|
|
941
|
+
s.packetWindow = [...previous.packetWindow, { received, lost }].slice(-3);
|
|
942
|
+
const totals = s.packetWindow.reduce((sum, interval) => ({
|
|
943
|
+
received: sum.received + interval.received,
|
|
944
|
+
lost: sum.lost + interval.lost
|
|
945
|
+
}), { received: 0, lost: 0 });
|
|
946
|
+
const packets = totals.received + totals.lost;
|
|
947
|
+
const fps = (sample.framesDecoded - before.framesDecoded) * 1e3 / elapsedMs;
|
|
948
|
+
const loss = packets >= p.minimumPackets ? totals.lost / packets : null;
|
|
949
|
+
const impaired = options.paused || fps < p.impairedFps;
|
|
950
|
+
const bad = options.paused || impaired && (loss !== null && loss >= p.lossRatio || received >= p.minimumPackets && sample.jitter >= p.jitterSeconds);
|
|
951
|
+
const clean = received > 0 && !options.paused && loss !== null && loss < p.lossRatio && sample.jitter < p.jitterSeconds && fps >= p.impairedFps;
|
|
952
|
+
const duration = Math.min(pollMs, elapsedMs);
|
|
953
|
+
s.badMs = bad ? previous.badMs + duration : 0;
|
|
954
|
+
s.postDowngradeBadMs = bad && previous.cap !== "high" ? previous.postDowngradeBadMs + duration : 0;
|
|
955
|
+
s.cleanMs = clean ? previous.cleanMs + duration : 0;
|
|
956
|
+
s.reducedBadMs = bad && fps === 0 && previous.cap !== "high" ? previous.reducedBadMs + duration : 0;
|
|
957
|
+
if (s.badMs >= p.downgradeMs && s.cap === "high") s.cap = "reduced";
|
|
958
|
+
else if (s.reducedBadMs >= p.floorMs && fps === 0 && s.cap === "reduced") s.cap = "floor";
|
|
959
|
+
if (s.postDowngradeBadMs >= p.notifyAfterDowngradeMs) s.status = "poor";
|
|
960
|
+
if (s.cleanMs >= p.recoverMs) s.cap = "high";
|
|
961
|
+
if (s.cleanMs >= p.clearNoticeMs) s.status = "healthy";
|
|
962
|
+
return s;
|
|
963
|
+
}
|
|
964
|
+
|
|
965
|
+
// ../client/src/react/network-quality-binding.ts
|
|
966
|
+
function bindNetworkQuality(publication, inhibited, notify, tickMs) {
|
|
967
|
+
let disposed = false;
|
|
968
|
+
let running = false;
|
|
969
|
+
let state = initNetworkQuality(performance.now());
|
|
970
|
+
const apply = () => {
|
|
971
|
+
const qualities = (publication.trackInfo?.layers ?? []).map((l) => l.quality);
|
|
972
|
+
const cap = state.cap === "high" ? VideoQuality.HIGH : resolveLowCapQuality(qualities, state.cap === "floor" ? 2 : 0);
|
|
973
|
+
try {
|
|
974
|
+
publication.setVideoQuality(cap);
|
|
975
|
+
} catch {
|
|
976
|
+
}
|
|
977
|
+
};
|
|
978
|
+
const emit = (status) => {
|
|
979
|
+
try {
|
|
980
|
+
notify(status);
|
|
981
|
+
} catch {
|
|
982
|
+
}
|
|
983
|
+
};
|
|
984
|
+
apply();
|
|
985
|
+
emit("unknown");
|
|
986
|
+
const tick = async () => {
|
|
987
|
+
if (disposed || running) return;
|
|
988
|
+
running = true;
|
|
989
|
+
try {
|
|
990
|
+
const track = publication.track;
|
|
991
|
+
const stats = await track?.getRTCStatsReport?.();
|
|
992
|
+
if (disposed || publication.track !== track) return;
|
|
993
|
+
const prior = state;
|
|
994
|
+
state = stepNetworkQuality(state, readNetworkVideoSample(stats), performance.now(), {
|
|
995
|
+
paused: track?.streamState === Track.StreamState.Paused,
|
|
996
|
+
inhibited: document.hidden || publication.isMuted || inhibited()
|
|
997
|
+
});
|
|
998
|
+
if (state.cap !== prior.cap) apply();
|
|
999
|
+
if (state.status !== prior.status) emit(state.status);
|
|
1000
|
+
} catch {
|
|
1001
|
+
if (!disposed) state = { ...state, sample: null, packetWindow: [], badMs: 0, postDowngradeBadMs: 0, cleanMs: 0, reducedBadMs: 0 };
|
|
1002
|
+
} finally {
|
|
1003
|
+
running = false;
|
|
1004
|
+
}
|
|
1005
|
+
};
|
|
1006
|
+
const timer = setInterval(() => {
|
|
1007
|
+
void tick();
|
|
1008
|
+
}, tickMs);
|
|
1009
|
+
return () => {
|
|
1010
|
+
disposed = true;
|
|
1011
|
+
clearInterval(timer);
|
|
1012
|
+
emit("unknown");
|
|
1013
|
+
};
|
|
1014
|
+
}
|
|
1015
|
+
|
|
1016
|
+
// ../client/src/react/use-quality-governor.ts
|
|
895
1017
|
function useAvatarQualityGovernor(input) {
|
|
896
|
-
const { enabled, freezeReading, config: policy = DEFAULT_GOVERNOR_CONFIG, tickMs = 1e3 } = input;
|
|
1018
|
+
const { enabled, freezeReading, onNetworkStatusChange, config: policy = DEFAULT_GOVERNOR_CONFIG, tickMs = 1e3 } = input;
|
|
897
1019
|
const room = useMaybeRoomContext();
|
|
898
1020
|
const { videoTrack } = useVoiceAssistant();
|
|
899
1021
|
const {
|
|
@@ -930,24 +1052,39 @@ function useAvatarQualityGovernor(input) {
|
|
|
930
1052
|
]);
|
|
931
1053
|
const freezeReadingRef = useRef(freezeReading);
|
|
932
1054
|
freezeReadingRef.current = freezeReading;
|
|
1055
|
+
const networkCallback = useRef(onNetworkStatusChange);
|
|
1056
|
+
networkCallback.current = onNetworkStatusChange;
|
|
933
1057
|
const targetPublication = videoTrack?.publication;
|
|
934
1058
|
const targetParticipant = videoTrack?.participant;
|
|
935
1059
|
const targetTrack = targetPublication?.track;
|
|
936
1060
|
useEffect(() => {
|
|
937
|
-
if (!
|
|
1061
|
+
if (!room || !targetPublication || !targetParticipant) return;
|
|
1062
|
+
if (enabled === "network-only") {
|
|
1063
|
+
return bindNetworkQuality(
|
|
1064
|
+
targetPublication,
|
|
1065
|
+
() => freezeReadingRef.current?.().inhibited ?? false,
|
|
1066
|
+
(status) => networkCallback.current?.(status),
|
|
1067
|
+
tickMs
|
|
1068
|
+
);
|
|
1069
|
+
}
|
|
1070
|
+
if (!enabled) {
|
|
1071
|
+
try {
|
|
1072
|
+
targetPublication.setVideoQuality(VideoQuality.HIGH);
|
|
1073
|
+
} catch {
|
|
1074
|
+
}
|
|
1075
|
+
return;
|
|
1076
|
+
}
|
|
938
1077
|
let pausedSinceTick = false;
|
|
939
1078
|
let lastFreezeStat = null;
|
|
940
1079
|
let jitterTrend = null;
|
|
941
|
-
let connQuality =
|
|
942
|
-
targetParticipant?.connectionQuality ?? ConnectionQuality.Unknown
|
|
943
|
-
);
|
|
1080
|
+
let connQuality = targetParticipant?.connectionQuality ?? ConnectionQuality.Unknown;
|
|
944
1081
|
const onStreamState = (pub, streamState) => {
|
|
945
1082
|
if (pub !== targetPublication) return;
|
|
946
1083
|
if (streamState === Track.StreamState.Paused) pausedSinceTick = true;
|
|
947
1084
|
};
|
|
948
1085
|
const onQuality = (q, participant) => {
|
|
949
1086
|
if (participant.sid !== targetParticipant.sid) return;
|
|
950
|
-
connQuality =
|
|
1087
|
+
connQuality = q;
|
|
951
1088
|
};
|
|
952
1089
|
try {
|
|
953
1090
|
room.on(RoomEvent.TrackStreamStateChanged, onStreamState);
|
package/dist/react.d.ts
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
import { RegisteredTool, ToolRegistration } from './tools.js';
|
|
2
2
|
import { ReactNode } from 'react';
|
|
3
|
-
import { g as AvatarSessionClient, A as AvatarVideoFit } from './proxy-client-
|
|
4
|
-
export { a as AdaptivePlayoutController, b as AdaptivePlayoutDecision, c as AdaptivePlayoutOptions, d as AdaptivePlayoutSample, e as ApproachingEndEvent, f as ApproachingEndReason, aK as AvatarVideoSurface, aL as AvatarVideoSurfaceProps, B as BehaviorSnapshot, C as CallTranscript, h as CallTranscriptSegment, i as CartesiaTtsModel, j as ClipResult, k as ClosingTurnResult, l as CreditsLowEvent, D as DEFAULT_APPROACHING_END_LEAD_SECONDS, m as DEFAULT_CREDITS_LOW_LEAD_SECONDS, n as DEFAULT_GOVERNOR_CONFIG, o as DEFAULT_GRACE_CEILING_SECONDS, p as DEFAULT_GRACE_WINDOW_LEAD_SECONDS, q as DEFAULT_IDLE_SECONDS, r as DEFAULT_IDLE_WARN_LEAD_SECONDS, s as DEFAULT_TURN_TIMEOUT_SECONDS, E as EndReason, t as EndedEvent, u as ExtendResult, F as FishTtsModel, v as FreezeReadingFn, G as Governor, w as GovernorAction, x as GovernorConfig, y as GovernorSignal, z as GovernorState, H as GraceWindowClosedEvent, I as GraceWindowOpenEvent, J as GraceWindowState, K as IdleWarningEvent, M as InboundRtpCursor, N as InboundRtpReading, O as KnownBehaviorState, P as LLMProvider, Q as LLMSelection, R as LiveKitAvatarGrantState, S as LiveKitAvatarGrantStatus, T as LiveKitCapacityState, U as LiveKitConnectionStatus, V as LiveKitSessionRequest, W as LiveKitSessionStartResult, aM as LivePlaybackKeeper, X as MAX_RECONNECT_ATTEMPTS, Y as MAX_SESSION_INSTRUCTIONS_CHARS, aN as MIC_LEASE_ENDED_TIMEOUT_MS, aO as MicLease, aP as PlayableVideoElement, Z as ProxyClientOptions, _ as QualityCap, $ as RECONNECT_BACKOFF_MS,
|
|
3
|
+
import { g as AvatarSessionClient, A as AvatarVideoFit } from './proxy-client-DhVdbfQX.js';
|
|
4
|
+
export { a as AdaptivePlayoutController, b as AdaptivePlayoutDecision, c as AdaptivePlayoutOptions, d as AdaptivePlayoutSample, e as ApproachingEndEvent, f as ApproachingEndReason, aK as AvatarVideoSurface, aL as AvatarVideoSurfaceProps, B as BehaviorSnapshot, C as CallTranscript, h as CallTranscriptSegment, i as CartesiaTtsModel, j as ClipResult, k as ClosingTurnResult, l as CreditsLowEvent, D as DEFAULT_APPROACHING_END_LEAD_SECONDS, m as DEFAULT_CREDITS_LOW_LEAD_SECONDS, n as DEFAULT_GOVERNOR_CONFIG, o as DEFAULT_GRACE_CEILING_SECONDS, p as DEFAULT_GRACE_WINDOW_LEAD_SECONDS, q as DEFAULT_IDLE_SECONDS, r as DEFAULT_IDLE_WARN_LEAD_SECONDS, s as DEFAULT_TURN_TIMEOUT_SECONDS, E as EndReason, t as EndedEvent, u as ExtendResult, F as FishTtsModel, v as FreezeReadingFn, G as Governor, w as GovernorAction, x as GovernorConfig, y as GovernorSignal, z as GovernorState, H as GraceWindowClosedEvent, I as GraceWindowOpenEvent, J as GraceWindowState, K as IdleWarningEvent, M as InboundRtpCursor, N as InboundRtpReading, O as KnownBehaviorState, P as LLMProvider, Q as LLMSelection, R as LiveKitAvatarGrantState, S as LiveKitAvatarGrantStatus, T as LiveKitCapacityState, U as LiveKitConnectionStatus, V as LiveKitSessionRequest, W as LiveKitSessionStartResult, aM as LivePlaybackKeeper, X as MAX_RECONNECT_ATTEMPTS, Y as MAX_SESSION_INSTRUCTIONS_CHARS, aN as MIC_LEASE_ENDED_TIMEOUT_MS, aO as MicLease, aP as NETWORK_QUALITY_POLICY, aQ as NetworkQualityStatus, aR as PlayableVideoElement, Z as ProxyClientOptions, _ as QualityCap, $ as RECONNECT_BACKOFF_MS, aS as RealtimeAvatarCapacityError, aT as RealtimeAvatarLiveKitRoom, aU as RealtimeAvatarLiveKitRoomProps, a0 as RealtimeAvatarRequestOptions, a1 as RealtimeSessionApi, a2 as RealtimeSessionMedia, a3 as RealtimeSessionRoomSinks, a4 as ReconnectPolicy, a5 as ReconnectingEvent, a6 as RecoveryState, a7 as RetryStep, a8 as SessionBehavior, a9 as SessionClip, aa as SessionClocks, ab as SessionEndReason, ac as SessionLifecycleApi, ad as SessionLifecyclePhase, ae as SessionLifecyclePhaseKind, af as SessionLifecycleRoomBridge, ag as SessionLifecycleRoomBridgeProps, ah as SurfaceLayers, ai as TurnState, aj as TurnTimeoutEvent, ak as UseAvatarQualityGovernorInput, al as UseLiveKitAvatarGrantInput, am as UseRealtimeSessionInput, an as UseSessionLifecycleInput, ao as VoiceSpec, ap as VoiceSpecInput, aq as capacityErrorFromBusy, ar as capacityStateFromGrant, as as createProxyClient, au as knownBehaviorStates, av as mapTurnState, aw as readInboundRtp, ax as sessionBehaviorSchema, ay as sessionClipSchema, aV as shouldReplayPendingTurn, az as splitCallTranscript, aA as useAvatarAdaptivePlayoutDelay, aB as useAvatarPlayoutDelay, aC as useAvatarQualityGovernor, aD as useCallTranscript, aE as useLiveKitAvatarGrant, aG as useMicLease, aH as useRealtimeSession, aI as useReleaseMicLeaseOnTrackEnded, aJ as useSessionLifecycle } from './proxy-client-DhVdbfQX.js';
|
|
5
5
|
export { D as DEFAULT_AVATAR_PLAYOUT_DELAY_SECONDS } from './playout-delay-DWOc8iCl.js';
|
|
6
6
|
import 'zod';
|
|
7
7
|
import '@livekit/components-react';
|
package/dist/react.js
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
"use client";
|
|
2
2
|
import { useRef, useState, useEffect, useCallback, useMemo, createElement, Fragment } from 'react';
|
|
3
3
|
import { useRoomContext, useConnectionState, useTranscriptions, useLocalParticipant, RoomAudioRenderer, LiveKitRoom, useMaybeRoomContext, useVoiceAssistant, VideoTrack, useChat } from '@livekit/components-react';
|
|
4
|
-
import { ConnectionState, ConnectionQuality, RoomEvent, TrackEvent, Track,
|
|
4
|
+
import { ConnectionState, isRemoteTrack, VideoQuality, ConnectionQuality, RoomEvent, TrackEvent, Track, ConnectionError, ConnectionErrorReason, DisconnectReason, Room } from 'livekit-client';
|
|
5
5
|
import { z } from 'zod';
|
|
6
6
|
|
|
7
7
|
// ../client/src/react/use-character-tools.ts
|
|
@@ -880,10 +880,8 @@ function useAvatarAdaptivePlayoutDelay(videoTrack, audioTrack, enabled = false,
|
|
|
880
880
|
const audioMediaTrack = audioTrack?.publication?.track;
|
|
881
881
|
useEffect(() => {
|
|
882
882
|
if (!enabled || !videoMediaTrack && !audioMediaTrack) return;
|
|
883
|
-
const
|
|
884
|
-
|
|
885
|
-
);
|
|
886
|
-
if (receivers.length === 0) return;
|
|
883
|
+
const tracks = [videoMediaTrack, audioMediaTrack].filter(isRemoteTrack);
|
|
884
|
+
if (tracks.length === 0) return;
|
|
887
885
|
const controller = new AdaptivePlayoutController(options);
|
|
888
886
|
const cursors = /* @__PURE__ */ new Map();
|
|
889
887
|
let closed = false;
|
|
@@ -893,9 +891,9 @@ function useAvatarAdaptivePlayoutDelay(videoTrack, audioTrack, enabled = false,
|
|
|
893
891
|
inFlight = true;
|
|
894
892
|
try {
|
|
895
893
|
let worst;
|
|
896
|
-
for (let i = 0; i <
|
|
897
|
-
const stats = await
|
|
898
|
-
if (closed ||
|
|
894
|
+
for (let i = 0; i < tracks.length; i++) {
|
|
895
|
+
const stats = await tracks[i].getRTCStatsReport();
|
|
896
|
+
if (closed || !stats) continue;
|
|
899
897
|
const reading = readInboundRtp(stats.values(), cursors.get(i));
|
|
900
898
|
if (!reading) continue;
|
|
901
899
|
cursors.set(i, reading.cursor);
|
|
@@ -1078,23 +1076,148 @@ var resolveLowCapQuality = (declaredLayerQualities, lowUnhealthy = 0) => {
|
|
|
1078
1076
|
return lowUnhealthy >= LOW_CAP_STEP_AFTER_UNHEALTHY ? sorted[0] : sorted[sorted.length - 2];
|
|
1079
1077
|
};
|
|
1080
1078
|
|
|
1081
|
-
// ../client/src/react/
|
|
1082
|
-
var
|
|
1083
|
-
|
|
1084
|
-
|
|
1085
|
-
|
|
1086
|
-
|
|
1087
|
-
|
|
1088
|
-
|
|
1089
|
-
|
|
1090
|
-
|
|
1091
|
-
|
|
1092
|
-
|
|
1093
|
-
|
|
1094
|
-
|
|
1079
|
+
// ../client/src/react/network-quality.ts
|
|
1080
|
+
var NETWORK_QUALITY_POLICY = {
|
|
1081
|
+
startupGraceMs: 3e3,
|
|
1082
|
+
maxSampleGapMs: 1500,
|
|
1083
|
+
minimumPackets: 20,
|
|
1084
|
+
lossWindowSamples: 3,
|
|
1085
|
+
lossRatio: 0.05,
|
|
1086
|
+
jitterSeconds: 0.1,
|
|
1087
|
+
impairedFps: 12,
|
|
1088
|
+
downgradeMs: 2e3,
|
|
1089
|
+
floorMs: 2e3,
|
|
1090
|
+
notifyAfterDowngradeMs: 5e3,
|
|
1091
|
+
recoverMs: 5e3,
|
|
1092
|
+
clearNoticeMs: 1e4
|
|
1095
1093
|
};
|
|
1094
|
+
function readNetworkVideoSample(report) {
|
|
1095
|
+
const samples = [];
|
|
1096
|
+
report?.forEach((row) => {
|
|
1097
|
+
if (row.type !== "inbound-rtp" || (row.kind ?? row.mediaType) !== "video") return;
|
|
1098
|
+
const { timestamp, framesDecoded, packetsReceived, packetsLost, jitter } = row;
|
|
1099
|
+
if (typeof row.id !== "string" || ![timestamp, framesDecoded, packetsReceived, packetsLost, jitter].every(
|
|
1100
|
+
(n) => typeof n === "number" && Number.isFinite(n)
|
|
1101
|
+
) || timestamp < 0 || framesDecoded < 0 || packetsReceived < 0 || jitter < 0) return;
|
|
1102
|
+
samples.push({
|
|
1103
|
+
id: `${row.id}:${row.ssrc ?? ""}`,
|
|
1104
|
+
timestamp,
|
|
1105
|
+
framesDecoded,
|
|
1106
|
+
packetsReceived,
|
|
1107
|
+
packetsLost,
|
|
1108
|
+
jitter
|
|
1109
|
+
});
|
|
1110
|
+
});
|
|
1111
|
+
return samples.length === 1 ? samples[0] : null;
|
|
1112
|
+
}
|
|
1113
|
+
function initNetworkQuality(nowMs) {
|
|
1114
|
+
return {
|
|
1115
|
+
cap: "high",
|
|
1116
|
+
status: "unknown",
|
|
1117
|
+
sample: null,
|
|
1118
|
+
lastPollMs: null,
|
|
1119
|
+
openedMs: nowMs,
|
|
1120
|
+
badMs: 0,
|
|
1121
|
+
postDowngradeBadMs: 0,
|
|
1122
|
+
reducedBadMs: 0,
|
|
1123
|
+
cleanMs: 0,
|
|
1124
|
+
packetWindow: []
|
|
1125
|
+
};
|
|
1126
|
+
}
|
|
1127
|
+
function stepNetworkQuality(previous, sample, nowMs, options) {
|
|
1128
|
+
const p = NETWORK_QUALITY_POLICY;
|
|
1129
|
+
const s = { ...previous, sample, lastPollMs: nowMs };
|
|
1130
|
+
const before = previous.sample;
|
|
1131
|
+
const pollMs = previous.lastPollMs === null ? 0 : nowMs - previous.lastPollMs;
|
|
1132
|
+
const elapsedMs = sample && before ? sample.timestamp - before.timestamp : 0;
|
|
1133
|
+
const interrupted = options.inhibited || pollMs <= 0 || pollMs > p.maxSampleGapMs;
|
|
1134
|
+
const valid = !interrupted && sample !== null && before !== null && sample.id === before.id && elapsedMs > 0 && elapsedMs <= p.maxSampleGapMs && sample.framesDecoded >= before.framesDecoded && sample.packetsReceived >= before.packetsReceived;
|
|
1135
|
+
if (!valid || nowMs - s.openedMs < p.startupGraceMs) {
|
|
1136
|
+
s.badMs = s.postDowngradeBadMs = s.reducedBadMs = s.cleanMs = 0;
|
|
1137
|
+
s.packetWindow = [];
|
|
1138
|
+
if (options.inhibited) s.sample = null;
|
|
1139
|
+
return s;
|
|
1140
|
+
}
|
|
1141
|
+
const received = sample.packetsReceived - before.packetsReceived;
|
|
1142
|
+
const lost = Math.max(0, sample.packetsLost - before.packetsLost);
|
|
1143
|
+
s.packetWindow = [...previous.packetWindow, { received, lost }].slice(-p.lossWindowSamples);
|
|
1144
|
+
const totals = s.packetWindow.reduce((sum, interval) => ({
|
|
1145
|
+
received: sum.received + interval.received,
|
|
1146
|
+
lost: sum.lost + interval.lost
|
|
1147
|
+
}), { received: 0, lost: 0 });
|
|
1148
|
+
const packets = totals.received + totals.lost;
|
|
1149
|
+
const fps = (sample.framesDecoded - before.framesDecoded) * 1e3 / elapsedMs;
|
|
1150
|
+
const loss = packets >= p.minimumPackets ? totals.lost / packets : null;
|
|
1151
|
+
const impaired = options.paused || fps < p.impairedFps;
|
|
1152
|
+
const bad = options.paused || impaired && (loss !== null && loss >= p.lossRatio || received >= p.minimumPackets && sample.jitter >= p.jitterSeconds);
|
|
1153
|
+
const clean = received > 0 && !options.paused && loss !== null && loss < p.lossRatio && sample.jitter < p.jitterSeconds && fps >= p.impairedFps;
|
|
1154
|
+
const duration = Math.min(pollMs, elapsedMs);
|
|
1155
|
+
s.badMs = bad ? previous.badMs + duration : 0;
|
|
1156
|
+
s.postDowngradeBadMs = bad && previous.cap !== "high" ? previous.postDowngradeBadMs + duration : 0;
|
|
1157
|
+
s.cleanMs = clean ? previous.cleanMs + duration : 0;
|
|
1158
|
+
s.reducedBadMs = bad && fps === 0 && previous.cap !== "high" ? previous.reducedBadMs + duration : 0;
|
|
1159
|
+
if (s.badMs >= p.downgradeMs && s.cap === "high") s.cap = "reduced";
|
|
1160
|
+
else if (s.reducedBadMs >= p.floorMs && fps === 0 && s.cap === "reduced") s.cap = "floor";
|
|
1161
|
+
if (s.postDowngradeBadMs >= p.notifyAfterDowngradeMs) s.status = "poor";
|
|
1162
|
+
if (s.cleanMs >= p.recoverMs) s.cap = "high";
|
|
1163
|
+
if (s.cleanMs >= p.clearNoticeMs) s.status = "healthy";
|
|
1164
|
+
return s;
|
|
1165
|
+
}
|
|
1166
|
+
|
|
1167
|
+
// ../client/src/react/network-quality-binding.ts
|
|
1168
|
+
function bindNetworkQuality(publication, inhibited, notify, tickMs) {
|
|
1169
|
+
let disposed = false;
|
|
1170
|
+
let running = false;
|
|
1171
|
+
let state = initNetworkQuality(performance.now());
|
|
1172
|
+
const apply = () => {
|
|
1173
|
+
const qualities = (publication.trackInfo?.layers ?? []).map((l) => l.quality);
|
|
1174
|
+
const cap = state.cap === "high" ? VideoQuality.HIGH : resolveLowCapQuality(qualities, state.cap === "floor" ? 2 : 0);
|
|
1175
|
+
try {
|
|
1176
|
+
publication.setVideoQuality(cap);
|
|
1177
|
+
} catch {
|
|
1178
|
+
}
|
|
1179
|
+
};
|
|
1180
|
+
const emit = (status) => {
|
|
1181
|
+
try {
|
|
1182
|
+
notify(status);
|
|
1183
|
+
} catch {
|
|
1184
|
+
}
|
|
1185
|
+
};
|
|
1186
|
+
apply();
|
|
1187
|
+
emit("unknown");
|
|
1188
|
+
const tick = async () => {
|
|
1189
|
+
if (disposed || running) return;
|
|
1190
|
+
running = true;
|
|
1191
|
+
try {
|
|
1192
|
+
const track = publication.track;
|
|
1193
|
+
const stats = await track?.getRTCStatsReport?.();
|
|
1194
|
+
if (disposed || publication.track !== track) return;
|
|
1195
|
+
const prior = state;
|
|
1196
|
+
state = stepNetworkQuality(state, readNetworkVideoSample(stats), performance.now(), {
|
|
1197
|
+
paused: track?.streamState === Track.StreamState.Paused,
|
|
1198
|
+
inhibited: document.hidden || publication.isMuted || inhibited()
|
|
1199
|
+
});
|
|
1200
|
+
if (state.cap !== prior.cap) apply();
|
|
1201
|
+
if (state.status !== prior.status) emit(state.status);
|
|
1202
|
+
} catch {
|
|
1203
|
+
if (!disposed) state = { ...state, sample: null, packetWindow: [], badMs: 0, postDowngradeBadMs: 0, cleanMs: 0, reducedBadMs: 0 };
|
|
1204
|
+
} finally {
|
|
1205
|
+
running = false;
|
|
1206
|
+
}
|
|
1207
|
+
};
|
|
1208
|
+
const timer = setInterval(() => {
|
|
1209
|
+
void tick();
|
|
1210
|
+
}, tickMs);
|
|
1211
|
+
return () => {
|
|
1212
|
+
disposed = true;
|
|
1213
|
+
clearInterval(timer);
|
|
1214
|
+
emit("unknown");
|
|
1215
|
+
};
|
|
1216
|
+
}
|
|
1217
|
+
|
|
1218
|
+
// ../client/src/react/use-quality-governor.ts
|
|
1096
1219
|
function useAvatarQualityGovernor(input) {
|
|
1097
|
-
const { enabled, freezeReading, config: policy = DEFAULT_GOVERNOR_CONFIG, tickMs = 1e3 } = input;
|
|
1220
|
+
const { enabled, freezeReading, onNetworkStatusChange, config: policy = DEFAULT_GOVERNOR_CONFIG, tickMs = 1e3 } = input;
|
|
1098
1221
|
const room = useMaybeRoomContext();
|
|
1099
1222
|
const { videoTrack } = useVoiceAssistant();
|
|
1100
1223
|
const {
|
|
@@ -1131,24 +1254,39 @@ function useAvatarQualityGovernor(input) {
|
|
|
1131
1254
|
]);
|
|
1132
1255
|
const freezeReadingRef = useRef(freezeReading);
|
|
1133
1256
|
freezeReadingRef.current = freezeReading;
|
|
1257
|
+
const networkCallback = useRef(onNetworkStatusChange);
|
|
1258
|
+
networkCallback.current = onNetworkStatusChange;
|
|
1134
1259
|
const targetPublication = videoTrack?.publication;
|
|
1135
1260
|
const targetParticipant = videoTrack?.participant;
|
|
1136
1261
|
const targetTrack = targetPublication?.track;
|
|
1137
1262
|
useEffect(() => {
|
|
1138
|
-
if (!
|
|
1263
|
+
if (!room || !targetPublication || !targetParticipant) return;
|
|
1264
|
+
if (enabled === "network-only") {
|
|
1265
|
+
return bindNetworkQuality(
|
|
1266
|
+
targetPublication,
|
|
1267
|
+
() => freezeReadingRef.current?.().inhibited ?? false,
|
|
1268
|
+
(status) => networkCallback.current?.(status),
|
|
1269
|
+
tickMs
|
|
1270
|
+
);
|
|
1271
|
+
}
|
|
1272
|
+
if (!enabled) {
|
|
1273
|
+
try {
|
|
1274
|
+
targetPublication.setVideoQuality(VideoQuality.HIGH);
|
|
1275
|
+
} catch {
|
|
1276
|
+
}
|
|
1277
|
+
return;
|
|
1278
|
+
}
|
|
1139
1279
|
let pausedSinceTick = false;
|
|
1140
1280
|
let lastFreezeStat = null;
|
|
1141
1281
|
let jitterTrend = null;
|
|
1142
|
-
let connQuality =
|
|
1143
|
-
targetParticipant?.connectionQuality ?? ConnectionQuality.Unknown
|
|
1144
|
-
);
|
|
1282
|
+
let connQuality = targetParticipant?.connectionQuality ?? ConnectionQuality.Unknown;
|
|
1145
1283
|
const onStreamState = (pub, streamState) => {
|
|
1146
1284
|
if (pub !== targetPublication) return;
|
|
1147
1285
|
if (streamState === Track.StreamState.Paused) pausedSinceTick = true;
|
|
1148
1286
|
};
|
|
1149
1287
|
const onQuality = (q, participant) => {
|
|
1150
1288
|
if (participant.sid !== targetParticipant.sid) return;
|
|
1151
|
-
connQuality =
|
|
1289
|
+
connQuality = q;
|
|
1152
1290
|
};
|
|
1153
1291
|
try {
|
|
1154
1292
|
room.on(RoomEvent.TrackStreamStateChanged, onStreamState);
|
|
@@ -1367,6 +1505,7 @@ function AvatarVideoSurface(props) {
|
|
|
1367
1505
|
// Keep the governor alive across presentation-intent changes; resetting it on
|
|
1368
1506
|
// every turn would re-apply LOW and a normal short turn could never earn HIGH.
|
|
1369
1507
|
enabled: adaptiveQuality,
|
|
1508
|
+
onNetworkStatusChange: props.onNetworkStatusChange,
|
|
1370
1509
|
freezeReading: frameFlow.freezeReading,
|
|
1371
1510
|
config: governorConfig
|
|
1372
1511
|
});
|
|
@@ -3263,4 +3402,4 @@ function createProxyClient(options) {
|
|
|
3263
3402
|
};
|
|
3264
3403
|
}
|
|
3265
3404
|
|
|
3266
|
-
export { AdaptivePlayoutController, AvatarCall, AvatarVideoSurface, DEFAULT_APPROACHING_END_LEAD_SECONDS, DEFAULT_AVATAR_PLAYOUT_DELAY_SECONDS, DEFAULT_AVATAR_UNSTABLE_STALL_MS, DEFAULT_CREDITS_LOW_LEAD_SECONDS, DEFAULT_GOVERNOR_CONFIG, DEFAULT_GRACE_CEILING_SECONDS, DEFAULT_GRACE_WINDOW_LEAD_SECONDS, DEFAULT_IDLE_SECONDS, DEFAULT_IDLE_WARN_LEAD_SECONDS, DEFAULT_TURN_TIMEOUT_SECONDS, MAX_RECONNECT_ATTEMPTS, MAX_SESSION_INSTRUCTIONS_CHARS, MIC_LEASE_ENDED_TIMEOUT_MS, RECONNECT_BACKOFF_MS, RealtimeAvatarCapacityError, RealtimeAvatarLiveKitRoom, SessionLifecycleRoomBridge, capacityErrorFromBusy, capacityStateFromGrant, createProxyClient, knownBehaviorStates, mapTurnState, readInboundRtp, sessionBehaviorSchema, sessionClipSchema, shouldReplayPendingTurn, splitCallTranscript, useAvatarAdaptivePlayoutDelay, useAvatarCall, useAvatarPlayoutDelay, useAvatarQualityGovernor, useCallTranscript, useCharacterTools, useLiveKitAvatarGrant, useMicLease, useRealtimeSession, useReleaseMicLeaseOnTrackEnded, useSessionLifecycle };
|
|
3405
|
+
export { AdaptivePlayoutController, AvatarCall, AvatarVideoSurface, DEFAULT_APPROACHING_END_LEAD_SECONDS, DEFAULT_AVATAR_PLAYOUT_DELAY_SECONDS, DEFAULT_AVATAR_UNSTABLE_STALL_MS, DEFAULT_CREDITS_LOW_LEAD_SECONDS, DEFAULT_GOVERNOR_CONFIG, DEFAULT_GRACE_CEILING_SECONDS, DEFAULT_GRACE_WINDOW_LEAD_SECONDS, DEFAULT_IDLE_SECONDS, DEFAULT_IDLE_WARN_LEAD_SECONDS, DEFAULT_TURN_TIMEOUT_SECONDS, MAX_RECONNECT_ATTEMPTS, MAX_SESSION_INSTRUCTIONS_CHARS, MIC_LEASE_ENDED_TIMEOUT_MS, NETWORK_QUALITY_POLICY, RECONNECT_BACKOFF_MS, RealtimeAvatarCapacityError, RealtimeAvatarLiveKitRoom, SessionLifecycleRoomBridge, capacityErrorFromBusy, capacityStateFromGrant, createProxyClient, knownBehaviorStates, mapTurnState, readInboundRtp, sessionBehaviorSchema, sessionClipSchema, shouldReplayPendingTurn, splitCallTranscript, useAvatarAdaptivePlayoutDelay, useAvatarCall, useAvatarPlayoutDelay, useAvatarQualityGovernor, useCallTranscript, useCharacterTools, useLiveKitAvatarGrant, useMicLease, useRealtimeSession, useReleaseMicLeaseOnTrackEnded, useSessionLifecycle };
|
package/dist/server.js
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
export { RealtimeAvatar, RealtimeAvatarError, RealtimeAvatarHttpError, clipLibraryDeclarationSchema, isQueued, verifyTranscript } from './chunk-
|
|
1
|
+
export { RealtimeAvatar, RealtimeAvatarError, RealtimeAvatarHttpError, clipLibraryDeclarationSchema, isQueued, verifyTranscript } from './chunk-UZQXZH2Z.js';
|
package/dist/tanstack-start.js
CHANGED
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "realtime-avatar",
|
|
3
|
-
"version": "0.
|
|
3
|
+
"version": "0.14.0",
|
|
4
4
|
"description": "Realtime Avatar SDK. `realtime-avatar` and the route adapters hold your API key and are server-only; `realtime-avatar/react`, `/browser` and `/tools` never can — importing a server entry into a browser build throws at bundle time.",
|
|
5
5
|
"license": "MIT",
|
|
6
6
|
"author": "The Influence Company",
|