realtime-avatar 0.20.0 → 0.22.0

package/dist/{chunk-PKOWNCDT.js → chunk-N76M4MEU.js}

@@ -290,6 +290,28 @@ function isTransient(cause) {
   const name = cause?.name;
   return name === "TimeoutError" || name === "TypeError" || name === "FetchError";
 }
+var schema02 = z.string().min(2).max(64);
+var schema110 = z.array(schema02);
+var schema27 = z.string().regex(new RegExp("\\S")).min(1).max(160);
+var schema32 = z.array(schema27).max(1e3);
+var schema42 = z.strictObject({
+  "language_codes": schema110.optional(),
+  "custom_vocabulary": schema32.optional()
+});
+var transcriptionWireSchema = schema42;
+var transcriptionOptionsSchema = z.object({
+  languageCodes: transcriptionWireSchema.shape.language_codes,
+  customVocabulary: transcriptionWireSchema.shape.custom_vocabulary
+}).strict();
+function transcriptionToWire(input) {
+  const options = transcriptionOptionsSchema.parse(input);
+  return {
+    ...options.languageCodes !== void 0 ? { language_codes: options.languageCodes } : {},
+    ...options.customVocabulary !== void 0 ? { custom_vocabulary: options.customVocabulary } : {}
+  };
+}
+
+// ../client/src/wire.ts
 var DEFAULT_AVATAR_ID = "maria";
 var DEFAULT_BACKGROUND_ID = "plain_white";
 var sessionModeSchema = z.enum(["avatar", "voice"]);
@@ -437,6 +459,7 @@ z.object({
   source_kind: avatarSourceKindSchema.default("portrait"),
   source_video_url: nullableUrlSchema.optional(),
   stt_mode: liveKitSttModeSchema.default("server"),
+  transcription: transcriptionWireSchema.optional(),
   camera: z.boolean().optional(),
   video_cache_id: z.string().min(1).max(240).nullable().optional(),
   voice: voiceSpecSchema.nullable().optional(),
@@ -510,6 +533,7 @@ z.object({
   queueTicketId: z.string().min(1).max(160).optional(),
   roomName: z.string().min(1).max(160).optional(),
   sttMode: liveKitSttModeSchema.default("server"),
+  transcription: transcriptionOptionsSchema.optional(),
   camera: z.boolean().optional(),
   voice: voiceSpecSchema.nullable().optional(),
   voiceId: z.string().min(1).max(240).nullable().optional(),
@@ -621,7 +645,7 @@ z.enum([
 
 // ../http-client/src/client.ts
 var DEFAULT_BASE_URL = "https://realtimeavatar.ai/api/v1";
-var SDK_VERSION = "0.20.0";
+var SDK_VERSION = "0.22.0";
 var RealtimeAvatar = class {
   #apiKey;
   #baseUrl;
@@ -661,6 +685,7 @@ var RealtimeAvatar = class {
     };
     if (options.instructions !== void 0) body.instructions = options.instructions;
     if (options.camera !== void 0) body.camera = options.camera;
+    if (options.transcription !== void 0) body.transcription = transcriptionToWire(options.transcription);
     if (options.context !== void 0) {
       body.initial_context = options.context.map((m) => ({ role: m.role, content: m.content }));
     }

package/dist/{chunk-QT5UTARL.js → chunk-T3RZSVRI.js}

@@ -1,4 +1,4 @@
-import { RealtimeAvatar, isQueued, RealtimeAvatarHttpError } from './chunk-PKOWNCDT.js';
+import { RealtimeAvatar, isQueued, RealtimeAvatarHttpError } from './chunk-N76M4MEU.js';
 
 // ../proxy/src/config.ts
 var json = (body, status = 200) => new Response(JSON.stringify(body), {

package/dist/express.d.ts

@@ -1,5 +1,5 @@
-import { P as ProxyConfig } from './types-TaQCvMPh.js';
-import './types-D76SpA3g.js';
+import { P as ProxyConfig } from './types-k6pW-wKA.js';
+import './types-Cmto2uGs.js';
 import 'zod';
 
 type Expressish = {

package/dist/express.js

@@ -1,5 +1,5 @@
-import { createProxyHandler } from './chunk-QT5UTARL.js';
-import './chunk-PKOWNCDT.js';
+import { createProxyHandler } from './chunk-T3RZSVRI.js';
+import './chunk-N76M4MEU.js';
 import './chunk-MCIXU6XP.js';
 
 // ../proxy/src/express.ts

package/dist/hono.d.ts

@@ -1,5 +1,5 @@
-import { P as ProxyConfig } from './types-TaQCvMPh.js';
-import './types-D76SpA3g.js';
+import { P as ProxyConfig } from './types-k6pW-wKA.js';
+import './types-Cmto2uGs.js';
 import 'zod';
 
 /**

package/dist/hono.js

@@ -1,5 +1,5 @@
-import { createProxyHandler } from './chunk-QT5UTARL.js';
-import './chunk-PKOWNCDT.js';
+import { createProxyHandler } from './chunk-T3RZSVRI.js';
+import './chunk-N76M4MEU.js';
 import './chunk-MCIXU6XP.js';
 
 // ../proxy/src/hono.ts

package/dist/index.d.ts

@@ -1,5 +1,5 @@
-import { a as CallPolicy, C as CallMode, S as StartCallResult, E as EndCallOptions, A as Avatar, b as AvatarUpdate, c as AvatarSourceSwap, d as ClipLibraryDeclaration, e as ClipLibraryUpdate, L as LoopRedirect, f as ClipLibrary, g as AssetKind, h as Asset, i as ListSessionsOptions, U as UsageSessionPage, j as UsageSession, k as ListRecordingsQuery, l as ListRecordingsResponse, R as RecordingArtifact, m as RecordingAccessResponse, n as CreditBalance, T as TranscriptPayload } from './types-D76SpA3g.js';
-export { o as AvatarClip, p as CallConnection, q as CallQueued, r as ClipDeclaration, s as ClipLibraryPlan, t as ClipSource, u as ContextMessage, v as EndCallReason, w as RecordingArtifactStatus, x as RecordingMode, V as VideoPolicy, y as isQueued, z as listRecordingsResponseSchema, B as recordingAccessResponseSchema, D as recordingArtifactSchema, F as recordingModeSchema } from './types-D76SpA3g.js';
+import { a as CallPolicy, C as CallMode, S as StartCallResult, E as EndCallOptions, A as Avatar, b as AvatarUpdate, c as AvatarSourceSwap, d as ClipLibraryDeclaration, e as ClipLibraryUpdate, L as LoopRedirect, f as ClipLibrary, g as AssetKind, h as Asset, i as ListSessionsOptions, U as UsageSessionPage, j as UsageSession, k as ListRecordingsQuery, l as ListRecordingsResponse, R as RecordingArtifact, m as RecordingAccessResponse, n as CreditBalance, T as TranscriptPayload } from './types-Cmto2uGs.js';
+export { o as AvatarClip, p as CallConnection, q as CallQueued, r as ClipDeclaration, s as ClipLibraryPlan, t as ClipSource, u as ContextMessage, v as EndCallReason, w as RecordingArtifactStatus, x as RecordingMode, V as VideoPolicy, y as isQueued, z as listRecordingsResponseSchema, B as recordingAccessResponseSchema, D as recordingArtifactSchema, F as recordingModeSchema } from './types-Cmto2uGs.js';
 import { ConnectionHistoryResponse } from './connection-history.js';
 import { z } from 'zod';
 

package/dist/index.js

@@ -1,2 +1,2 @@
-export { RealtimeAvatar, RealtimeAvatarError, RealtimeAvatarHttpError, clipLibraryDeclarationSchema, isQueued, listRecordingsResponseSchema, recordingAccessResponseSchema, recordingArtifactSchema, recordingModeSchema, verifyTranscript } from './chunk-PKOWNCDT.js';
+export { RealtimeAvatar, RealtimeAvatarError, RealtimeAvatarHttpError, clipLibraryDeclarationSchema, isQueued, listRecordingsResponseSchema, recordingAccessResponseSchema, recordingArtifactSchema, recordingModeSchema, verifyTranscript } from './chunk-N76M4MEU.js';
 import './chunk-MCIXU6XP.js';

package/dist/nextjs.d.ts

@@ -1,5 +1,5 @@
-import { P as ProxyConfig } from './types-TaQCvMPh.js';
-import './types-D76SpA3g.js';
+import { P as ProxyConfig } from './types-k6pW-wKA.js';
+import './types-Cmto2uGs.js';
 import 'zod';
 
 /**

package/dist/nextjs.js

@@ -1,5 +1,5 @@
-import { createProxyHandler } from './chunk-QT5UTARL.js';
-import './chunk-PKOWNCDT.js';
+import { createProxyHandler } from './chunk-T3RZSVRI.js';
+import './chunk-N76M4MEU.js';
 import './chunk-MCIXU6XP.js';
 
 // ../proxy/src/nextjs.ts

package/dist/{proxy-client-DpMcj6fI.d.ts → proxy-client-6ndCkiio.d.ts}

@@ -159,6 +159,10 @@ declare const liveKitSessionRequestSchema: z.ZodObject<{
         off: "off";
         server: "server";
     }>>;
+    transcription: z.ZodOptional<z.ZodObject<{
+        languageCodes: z.ZodOptional<z.ZodArray<z.ZodString>>;
+        customVocabulary: z.ZodOptional<z.ZodArray<z.ZodString>>;
+    }, z.core.$strict>>;
     camera: z.ZodOptional<z.ZodBoolean>;
     voice: z.ZodOptional<z.ZodNullable<z.ZodDiscriminatedUnion<[z.ZodObject<{
         provider: z.ZodLiteral<"cartesia">;
@@ -226,8 +230,8 @@ declare const liveKitSessionRequestSchema: z.ZodObject<{
             rules: z.ZodString;
             cooldown_seconds: z.ZodOptional<z.ZodNumber>;
             renderer: z.ZodOptional<z.ZodEnum<{
-                editor: "editor";
                 generative: "generative";
+                editor: "editor";
             }>>;
         }, z.core.$strict>>;
     }, z.core.$strict>>;
@@ -1375,6 +1379,12 @@ type AvatarConnectionDetails = Readonly<{
     video: PublisherConnectionDetails | null;
 }>;
 type SessionLifecycleRoomBridgeProps = {
+    /**
+     * Authoritative agent output, not the requested mode or a missing-track heuristic.
+     * Null means unknown (older worker / retired binding). Changing presentation must
+     * not re-mint the session. Shared by web and native; late joins read stored state.
+     */
+    onMediaModeChange?: (mode: "avatar" | "voice" | null) => void;
     /**
      * Opt in to an initial snapshot and changed facts. Null clears a retired binding.
      * Callback failures never affect the call; no stats polling or uploads are added.
@@ -1399,7 +1409,7 @@ type SessionLifecycleRoomBridgeProps = {
  *
  * Renders nothing. Mount it once inside RealtimeAvatarLiveKitRoom.
  */
-declare function SessionLifecycleRoomBridge({ lifecycle, onConnectionDetailsChange, }: SessionLifecycleRoomBridgeProps): null;
+declare function SessionLifecycleRoomBridge({ lifecycle, onConnectionDetailsChange, onMediaModeChange, }: SessionLifecycleRoomBridgeProps): null;
 
 /**
  * The terminal, LABELED end reason surfaced to the app via `onEnded`. This is the

package/dist/react-native.d.ts

@@ -1,7 +1,7 @@
 import { LiveKitRoomProps } from '@livekit/react-native';
 export { AudioSession, VideoTrack, VideoTrackProps, registerGlobals } from '@livekit/react-native';
-import { L as LiveKitSessionGrant, A as AvatarVideoFit } from './proxy-client-DpMcj6fI.js';
-export { a as AdaptivePlayoutController, b as AdaptivePlayoutDecision, c as AdaptivePlayoutOptions, d as AdaptivePlayoutSample, e as ApproachingEndEvent, f as ApproachingEndReason, g as AvatarConnectionDetails, h as AvatarSessionClient, B as BehaviorSnapshot, C as CallTranscript, i as CallTranscriptSegment, j as CartesiaTtsModel, k as ClipResult, l as ClosingTurnResult, m as CreditsLowEvent, D as DEFAULT_APPROACHING_END_LEAD_SECONDS, n as DEFAULT_CREDITS_LOW_LEAD_SECONDS, o as DEFAULT_GOVERNOR_CONFIG, p as DEFAULT_GRACE_CEILING_SECONDS, q as DEFAULT_GRACE_WINDOW_LEAD_SECONDS, r as DEFAULT_IDLE_SECONDS, s as DEFAULT_IDLE_WARN_LEAD_SECONDS, t as DEFAULT_TURN_TIMEOUT_SECONDS, E as EndReason, u as EndedEvent, v as ExtendResult, F as FishTtsModel, w as FreezeReadingFn, G as Governor, x as GovernorAction, y as GovernorConfig, z as GovernorSignal, H as GovernorState, I as GraceWindowClosedEvent, J as GraceWindowOpenEvent, K as GraceWindowState, M as IdleWarningEvent, N as InboundRtpCursor, O as InboundRtpReading, P as KnownBehaviorState, Q as LLMProvider, R as LLMSelection, S as LiveKitAvatarGrantState, T as LiveKitAvatarGrantStatus, U as LiveKitCapacityState, V as LiveKitConnectionStatus, W as LiveKitSessionRequest, X as LiveKitSessionStartResult, Y as MAX_RECONNECT_ATTEMPTS, Z as MAX_SESSION_INSTRUCTIONS_CHARS, _ as ProxyClientOptions, $ as QualityCap, a0 as RECONNECT_BACKOFF_MS, a1 as RealtimeAvatarRequestOptions, a2 as RealtimeSessionApi, a3 as RealtimeSessionMedia, a4 as RealtimeSessionRoomSinks, a5 as ReconnectPolicy, a6 as ReconnectingEvent, a7 as RecoveryState, a8 as RetryStep, a9 as SessionBehavior, aa as SessionClip, ab as SessionClocks, ac as SessionEndReason, ad as SessionLifecycleApi, ae as SessionLifecyclePhase, af as SessionLifecyclePhaseKind, ag as SessionLifecycleRoomBridge, ah as SessionLifecycleRoomBridgeProps, ai as SurfaceLayers, aj as TurnState, ak as TurnTimeoutEvent, al as UseAvatarQualityGovernorInput, am as UseLiveKitAvatarGrantInput, an as UseRealtimeSessionInput, ao as UseSessionLifecycleInput, ap as VoiceSpec, aq as VoiceSpecInput, ar as capacityErrorFromBusy, as as capacityStateFromGrant, at as createProxyClient, au as isNativeLiveTrackSubscribed, av as knownBehaviorStates, aw as mapTurnState, ax as readInboundRtp, ay as sessionBehaviorSchema, az as sessionClipSchema, aA as splitCallTranscript, aB as useAvatarAdaptivePlayoutDelay, aC as useAvatarCamera, aD as useAvatarPlayoutDelay, aE as useAvatarQualityGovernor, aF as useCallTranscript, aG as useLiveKitAvatarGrant, aH as useLiveTrackProducing, aI as useMicLease, aJ as useRealtimeSession, aK as useReleaseMicLeaseOnTrackEnded, aL as useSessionLifecycle } from './proxy-client-DpMcj6fI.js';
+import { L as LiveKitSessionGrant, A as AvatarVideoFit } from './proxy-client-6ndCkiio.js';
+export { a as AdaptivePlayoutController, b as AdaptivePlayoutDecision, c as AdaptivePlayoutOptions, d as AdaptivePlayoutSample, e as ApproachingEndEvent, f as ApproachingEndReason, g as AvatarConnectionDetails, h as AvatarSessionClient, B as BehaviorSnapshot, C as CallTranscript, i as CallTranscriptSegment, j as CartesiaTtsModel, k as ClipResult, l as ClosingTurnResult, m as CreditsLowEvent, D as DEFAULT_APPROACHING_END_LEAD_SECONDS, n as DEFAULT_CREDITS_LOW_LEAD_SECONDS, o as DEFAULT_GOVERNOR_CONFIG, p as DEFAULT_GRACE_CEILING_SECONDS, q as DEFAULT_GRACE_WINDOW_LEAD_SECONDS, r as DEFAULT_IDLE_SECONDS, s as DEFAULT_IDLE_WARN_LEAD_SECONDS, t as DEFAULT_TURN_TIMEOUT_SECONDS, E as EndReason, u as EndedEvent, v as ExtendResult, F as FishTtsModel, w as FreezeReadingFn, G as Governor, x as GovernorAction, y as GovernorConfig, z as GovernorSignal, H as GovernorState, I as GraceWindowClosedEvent, J as GraceWindowOpenEvent, K as GraceWindowState, M as IdleWarningEvent, N as InboundRtpCursor, O as InboundRtpReading, P as KnownBehaviorState, Q as LLMProvider, R as LLMSelection, S as LiveKitAvatarGrantState, T as LiveKitAvatarGrantStatus, U as LiveKitCapacityState, V as LiveKitConnectionStatus, W as LiveKitSessionRequest, X as LiveKitSessionStartResult, Y as MAX_RECONNECT_ATTEMPTS, Z as MAX_SESSION_INSTRUCTIONS_CHARS, _ as ProxyClientOptions, $ as QualityCap, a0 as RECONNECT_BACKOFF_MS, a1 as RealtimeAvatarRequestOptions, a2 as RealtimeSessionApi, a3 as RealtimeSessionMedia, a4 as RealtimeSessionRoomSinks, a5 as ReconnectPolicy, a6 as ReconnectingEvent, a7 as RecoveryState, a8 as RetryStep, a9 as SessionBehavior, aa as SessionClip, ab as SessionClocks, ac as SessionEndReason, ad as SessionLifecycleApi, ae as SessionLifecyclePhase, af as SessionLifecyclePhaseKind, ag as SessionLifecycleRoomBridge, ah as SessionLifecycleRoomBridgeProps, ai as SurfaceLayers, aj as TurnState, ak as TurnTimeoutEvent, al as UseAvatarQualityGovernorInput, am as UseLiveKitAvatarGrantInput, an as UseRealtimeSessionInput, ao as UseSessionLifecycleInput, ap as VoiceSpec, aq as VoiceSpecInput, ar as capacityErrorFromBusy, as as capacityStateFromGrant, at as createProxyClient, au as isNativeLiveTrackSubscribed, av as knownBehaviorStates, aw as mapTurnState, ax as readInboundRtp, ay as sessionBehaviorSchema, az as sessionClipSchema, aA as splitCallTranscript, aB as useAvatarAdaptivePlayoutDelay, aC as useAvatarCamera, aD as useAvatarPlayoutDelay, aE as useAvatarQualityGovernor, aF as useCallTranscript, aG as useLiveKitAvatarGrant, aH as useLiveTrackProducing, aI as useMicLease, aJ as useRealtimeSession, aK as useReleaseMicLeaseOnTrackEnded, aL as useSessionLifecycle } from './proxy-client-6ndCkiio.js';
 import { ReactNode, ReactElement } from 'react';
 import { StyleProp, ViewStyle } from 'react-native';
 export { useChat, useConnectionState, useLocalParticipant, useRoomContext, useTrackToggle, useTranscriptions, useVoiceAssistant } from '@livekit/components-react';

package/dist/react-native.js

@@ -1649,6 +1649,19 @@ z.object({
   sessionId: z.string().min(1).max(200),
   observations: z.array(connectionHistoryObservationSchema).max(MAX_CONNECTION_HISTORY_OBSERVATIONS)
 }).strict();
+var schema0 = z.string().min(2).max(64);
+var schema1 = z.array(schema0);
+var schema2 = z.string().regex(new RegExp("\\S")).min(1).max(160);
+var schema3 = z.array(schema2).max(1e3);
+var schema4 = z.strictObject({
+  "language_codes": schema1.optional(),
+  "custom_vocabulary": schema3.optional()
+});
+var transcriptionWireSchema = schema4;
+var transcriptionOptionsSchema = z.object({
+  languageCodes: transcriptionWireSchema.shape.language_codes,
+  customVocabulary: transcriptionWireSchema.shape.custom_vocabulary
+}).strict();
 
 // ../client/src/wire.ts
 var DEFAULT_AVATAR_ID = "maria";
@@ -1798,6 +1811,7 @@ z.object({
   source_kind: avatarSourceKindSchema.default("portrait"),
   source_video_url: nullableUrlSchema.optional(),
   stt_mode: liveKitSttModeSchema.default("server"),
+  transcription: transcriptionWireSchema.optional(),
   camera: z.boolean().optional(),
   video_cache_id: z.string().min(1).max(240).nullable().optional(),
   voice: voiceSpecSchema.nullable().optional(),
@@ -1871,6 +1885,7 @@ z.object({
   queueTicketId: z.string().min(1).max(160).optional(),
   roomName: z.string().min(1).max(160).optional(),
   sttMode: liveKitSttModeSchema.default("server"),
+  transcription: transcriptionOptionsSchema.optional(),
   camera: z.boolean().optional(),
   voice: voiceSpecSchema.nullable().optional(),
   voiceId: z.string().min(1).max(240).nullable().optional(),
@@ -2536,7 +2551,8 @@ function useSessionLifecycle(input) {
 }
 function SessionLifecycleRoomBridge({
   lifecycle,
-  onConnectionDetailsChange
+  onConnectionDetailsChange,
+  onMediaModeChange
 }) {
   const {
     onConnectionStateChange,
@@ -2553,6 +2569,37 @@ function SessionLifecycleRoomBridge({
   const assistant = useVoiceAssistant();
   const transcriptions = useTranscriptions();
   const room = useRoomContext();
+  const mediaModeCallbackRef = useRef(onMediaModeChange);
+  mediaModeCallbackRef.current = onMediaModeChange;
+  const mediaModeEnabled = onMediaModeChange !== void 0 && lifecycle.grant !== null;
+  const mediaModeSessionId = lifecycle.grant?.session_id;
+  useEffect(() => {
+    if (!mediaModeEnabled) return;
+    const agent = assistant.agent;
+    let previous;
+    const deliver = (mode) => {
+      if (mode === previous) return;
+      previous = mode;
+      try {
+        void Promise.resolve(mediaModeCallbackRef.current?.(mode)).catch(() => {
+        });
+      } catch {
+      }
+    };
+    const read = () => {
+      const value = agent?.attributes?.["rta.media_mode"];
+      deliver(value === "video" ? "avatar" : value === "voice" ? "voice" : null);
+    };
+    const changed = (_attributes, participant) => {
+      if (participant === agent) read();
+    };
+    room.on(RoomEvent.ParticipantAttributesChanged, changed);
+    read();
+    return () => {
+      room.off(RoomEvent.ParticipantAttributesChanged, changed);
+      deliver(null);
+    };
+  }, [room, assistant.agent, mediaModeEnabled, mediaModeSessionId]);
   const { send } = useChat();
   const agentPresent = Boolean(assistant.agent);
   const assistantState = assistant.state;

package/dist/react.d.ts

@@ -1,6 +1,6 @@
 import { RegisteredTool, ToolRegistration } from './tools.js';
-import { ah as SessionLifecycleRoomBridgeProps, h as AvatarSessionClient, A as AvatarVideoFit } from './proxy-client-DpMcj6fI.js';
-export { a as AdaptivePlayoutController, b as AdaptivePlayoutDecision, c as AdaptivePlayoutOptions, d as AdaptivePlayoutSample, e as ApproachingEndEvent, f as ApproachingEndReason, g as AvatarConnectionDetails, aM as AvatarVideoSurface, aN as AvatarVideoSurfaceProps, B as BehaviorSnapshot, C as CallTranscript, i as CallTranscriptSegment, j as CartesiaTtsModel, k as ClipResult, l as ClosingTurnResult, m as CreditsLowEvent, D as DEFAULT_APPROACHING_END_LEAD_SECONDS, n as DEFAULT_CREDITS_LOW_LEAD_SECONDS, o as DEFAULT_GOVERNOR_CONFIG, p as DEFAULT_GRACE_CEILING_SECONDS, q as DEFAULT_GRACE_WINDOW_LEAD_SECONDS, r as DEFAULT_IDLE_SECONDS, s as DEFAULT_IDLE_WARN_LEAD_SECONDS, t as DEFAULT_TURN_TIMEOUT_SECONDS, E as EndReason, u as EndedEvent, v as ExtendResult, F as FishTtsModel, w as FreezeReadingFn, G as Governor, x as GovernorAction, y as GovernorConfig, z as GovernorSignal, H as GovernorState, I as GraceWindowClosedEvent, J as GraceWindowOpenEvent, K as GraceWindowState, M as IdleWarningEvent, N as InboundRtpCursor, O as InboundRtpReading, P as KnownBehaviorState, Q as LLMProvider, R as LLMSelection, S as LiveKitAvatarGrantState, T as LiveKitAvatarGrantStatus, U as LiveKitCapacityState, V as LiveKitConnectionStatus, W as LiveKitSessionRequest, X as LiveKitSessionStartResult, aO as LivePlaybackKeeper, Y as MAX_RECONNECT_ATTEMPTS, Z as MAX_SESSION_INSTRUCTIONS_CHARS, aP as MIC_LEASE_ENDED_TIMEOUT_MS, aQ as MicLease, aR as NETWORK_QUALITY_POLICY, aS as NetworkQualityStatus, aT as PlayableVideoElement, _ as ProxyClientOptions, $ as QualityCap, a0 as RECONNECT_BACKOFF_MS, aU as RealtimeAvatarCapacityError, aV as RealtimeAvatarLiveKitRoom, aW as RealtimeAvatarLiveKitRoomProps, a1 as RealtimeAvatarRequestOptions, a2 as RealtimeSessionApi, a3 as RealtimeSessionMedia, a4 as RealtimeSessionRoomSinks, a5 as ReconnectPolicy, a6 as ReconnectingEvent, a7 as RecoveryState, a8 as RetryStep, a9 as SessionBehavior, aa as SessionClip, ab as SessionClocks, ac as SessionEndReason, ad as SessionLifecycleApi, ae as SessionLifecyclePhase, af as SessionLifecyclePhaseKind, ag as SessionLifecycleRoomBridge, ai as SurfaceLayers, aj as TurnState, ak as TurnTimeoutEvent, al as UseAvatarQualityGovernorInput, am as UseLiveKitAvatarGrantInput, an as UseRealtimeSessionInput, ao as UseSessionLifecycleInput, ap as VoiceSpec, aq as VoiceSpecInput, ar as capacityErrorFromBusy, as as capacityStateFromGrant, at as createProxyClient, av as knownBehaviorStates, aw as mapTurnState, ax as readInboundRtp, ay as sessionBehaviorSchema, az as sessionClipSchema, aX as shouldReplayPendingTurn, aA as splitCallTranscript, aB as useAvatarAdaptivePlayoutDelay, aC as useAvatarCamera, aD as useAvatarPlayoutDelay, aE as useAvatarQualityGovernor, aF as useCallTranscript, aG as useLiveKitAvatarGrant, aI as useMicLease, aJ as useRealtimeSession, aK as useReleaseMicLeaseOnTrackEnded, aL as useSessionLifecycle } from './proxy-client-DpMcj6fI.js';
+import { ah as SessionLifecycleRoomBridgeProps, h as AvatarSessionClient, A as AvatarVideoFit } from './proxy-client-6ndCkiio.js';
+export { a as AdaptivePlayoutController, b as AdaptivePlayoutDecision, c as AdaptivePlayoutOptions, d as AdaptivePlayoutSample, e as ApproachingEndEvent, f as ApproachingEndReason, g as AvatarConnectionDetails, aM as AvatarVideoSurface, aN as AvatarVideoSurfaceProps, B as BehaviorSnapshot, C as CallTranscript, i as CallTranscriptSegment, j as CartesiaTtsModel, k as ClipResult, l as ClosingTurnResult, m as CreditsLowEvent, D as DEFAULT_APPROACHING_END_LEAD_SECONDS, n as DEFAULT_CREDITS_LOW_LEAD_SECONDS, o as DEFAULT_GOVERNOR_CONFIG, p as DEFAULT_GRACE_CEILING_SECONDS, q as DEFAULT_GRACE_WINDOW_LEAD_SECONDS, r as DEFAULT_IDLE_SECONDS, s as DEFAULT_IDLE_WARN_LEAD_SECONDS, t as DEFAULT_TURN_TIMEOUT_SECONDS, E as EndReason, u as EndedEvent, v as ExtendResult, F as FishTtsModel, w as FreezeReadingFn, G as Governor, x as GovernorAction, y as GovernorConfig, z as GovernorSignal, H as GovernorState, I as GraceWindowClosedEvent, J as GraceWindowOpenEvent, K as GraceWindowState, M as IdleWarningEvent, N as InboundRtpCursor, O as InboundRtpReading, P as KnownBehaviorState, Q as LLMProvider, R as LLMSelection, S as LiveKitAvatarGrantState, T as LiveKitAvatarGrantStatus, U as LiveKitCapacityState, V as LiveKitConnectionStatus, W as LiveKitSessionRequest, X as LiveKitSessionStartResult, aO as LivePlaybackKeeper, Y as MAX_RECONNECT_ATTEMPTS, Z as MAX_SESSION_INSTRUCTIONS_CHARS, aP as MIC_LEASE_ENDED_TIMEOUT_MS, aQ as MicLease, aR as NETWORK_QUALITY_POLICY, aS as NetworkQualityStatus, aT as PlayableVideoElement, _ as ProxyClientOptions, $ as QualityCap, a0 as RECONNECT_BACKOFF_MS, aU as RealtimeAvatarCapacityError, aV as RealtimeAvatarLiveKitRoom, aW as RealtimeAvatarLiveKitRoomProps, a1 as RealtimeAvatarRequestOptions, a2 as RealtimeSessionApi, a3 as RealtimeSessionMedia, a4 as RealtimeSessionRoomSinks, a5 as ReconnectPolicy, a6 as ReconnectingEvent, a7 as RecoveryState, a8 as RetryStep, a9 as SessionBehavior, aa as SessionClip, ab as SessionClocks, ac as SessionEndReason, ad as SessionLifecycleApi, ae as SessionLifecyclePhase, af as SessionLifecyclePhaseKind, ag as SessionLifecycleRoomBridge, ai as SurfaceLayers, aj as TurnState, ak as TurnTimeoutEvent, al as UseAvatarQualityGovernorInput, am as UseLiveKitAvatarGrantInput, an as UseRealtimeSessionInput, ao as UseSessionLifecycleInput, ap as VoiceSpec, aq as VoiceSpecInput, ar as capacityErrorFromBusy, as as capacityStateFromGrant, at as createProxyClient, av as knownBehaviorStates, aw as mapTurnState, ax as readInboundRtp, ay as sessionBehaviorSchema, az as sessionClipSchema, aX as shouldReplayPendingTurn, aA as splitCallTranscript, aB as useAvatarAdaptivePlayoutDelay, aC as useAvatarCamera, aD as useAvatarPlayoutDelay, aE as useAvatarQualityGovernor, aF as useCallTranscript, aG as useLiveKitAvatarGrant, aI as useMicLease, aJ as useRealtimeSession, aK as useReleaseMicLeaseOnTrackEnded, aL as useSessionLifecycle } from './proxy-client-6ndCkiio.js';
 import { ReactNode } from 'react';
 export { D as DEFAULT_AVATAR_PLAYOUT_DELAY_SECONDS } from './playout-delay-DWOc8iCl.js';
 import 'zod';

package/dist/react.js

@@ -2183,6 +2183,19 @@ z.object({
   sessionId: z.string().min(1).max(200),
   observations: z.array(connectionHistoryObservationSchema).max(MAX_CONNECTION_HISTORY_OBSERVATIONS)
 }).strict();
+var schema0 = z.string().min(2).max(64);
+var schema1 = z.array(schema0);
+var schema2 = z.string().regex(new RegExp("\\S")).min(1).max(160);
+var schema3 = z.array(schema2).max(1e3);
+var schema4 = z.strictObject({
+  "language_codes": schema1.optional(),
+  "custom_vocabulary": schema3.optional()
+});
+var transcriptionWireSchema = schema4;
+var transcriptionOptionsSchema = z.object({
+  languageCodes: transcriptionWireSchema.shape.language_codes,
+  customVocabulary: transcriptionWireSchema.shape.custom_vocabulary
+}).strict();
 
 // ../client/src/wire.ts
 var DEFAULT_AVATAR_ID = "maria";
@@ -2332,6 +2345,7 @@ z.object({
   source_kind: avatarSourceKindSchema.default("portrait"),
   source_video_url: nullableUrlSchema.optional(),
   stt_mode: liveKitSttModeSchema.default("server"),
+  transcription: transcriptionWireSchema.optional(),
   camera: z.boolean().optional(),
   video_cache_id: z.string().min(1).max(240).nullable().optional(),
   voice: voiceSpecSchema.nullable().optional(),
@@ -2405,6 +2419,7 @@ z.object({
   queueTicketId: z.string().min(1).max(160).optional(),
   roomName: z.string().min(1).max(160).optional(),
   sttMode: liveKitSttModeSchema.default("server"),
+  transcription: transcriptionOptionsSchema.optional(),
   camera: z.boolean().optional(),
   voice: voiceSpecSchema.nullable().optional(),
   voiceId: z.string().min(1).max(240).nullable().optional(),
@@ -3075,7 +3090,8 @@ function useSessionLifecycle(input) {
 }
 function SessionLifecycleRoomBridge({
   lifecycle,
-  onConnectionDetailsChange
+  onConnectionDetailsChange,
+  onMediaModeChange
 }) {
   const {
     onConnectionStateChange,
@@ -3092,6 +3108,37 @@ function SessionLifecycleRoomBridge({
   const assistant = useVoiceAssistant();
   const transcriptions = useTranscriptions();
   const room = useRoomContext();
+  const mediaModeCallbackRef = useRef(onMediaModeChange);
+  mediaModeCallbackRef.current = onMediaModeChange;
+  const mediaModeEnabled = onMediaModeChange !== void 0 && lifecycle.grant !== null;
+  const mediaModeSessionId = lifecycle.grant?.session_id;
+  useEffect(() => {
+    if (!mediaModeEnabled) return;
+    const agent = assistant.agent;
+    let previous;
+    const deliver = (mode) => {
+      if (mode === previous) return;
+      previous = mode;
+      try {
+        void Promise.resolve(mediaModeCallbackRef.current?.(mode)).catch(() => {
+        });
+      } catch {
+      }
+    };
+    const read = () => {
+      const value = agent?.attributes?.["rta.media_mode"];
+      deliver(value === "video" ? "avatar" : value === "voice" ? "voice" : null);
+    };
+    const changed = (_attributes, participant) => {
+      if (participant === agent) read();
+    };
+    room.on(RoomEvent.ParticipantAttributesChanged, changed);
+    read();
+    return () => {
+      room.off(RoomEvent.ParticipantAttributesChanged, changed);
+      deliver(null);
+    };
+  }, [room, assistant.agent, mediaModeEnabled, mediaModeSessionId]);
   const { send } = useChat();
   const agentPresent = Boolean(assistant.agent);
   const assistantState = assistant.state;

package/dist/server.d.ts

@@ -1,4 +1,4 @@
 export { RealtimeAvatar, RealtimeAvatarError, RealtimeAvatarHttpError, RealtimeAvatarOptions, StartCallOptions, clipLibraryDeclarationSchema, verifyTranscript } from './index.js';
-export { h as Asset, g as AssetKind, A as Avatar, o as AvatarClip, c as AvatarSourceSwap, b as AvatarUpdate, p as CallConnection, C as CallMode, a as CallPolicy, q as CallQueued, r as ClipDeclaration, f as ClipLibrary, d as ClipLibraryDeclaration, s as ClipLibraryPlan, e as ClipLibraryUpdate, t as ClipSource, u as ContextMessage, n as CreditBalance, E as EndCallOptions, v as EndCallReason, k as ListRecordingsQuery, l as ListRecordingsResponse, i as ListSessionsOptions, L as LoopRedirect, m as RecordingAccessResponse, R as RecordingArtifact, w as RecordingArtifactStatus, x as RecordingMode, S as StartCallResult, T as TranscriptPayload, j as UsageSession, U as UsageSessionPage, V as VideoPolicy, y as isQueued, z as listRecordingsResponseSchema, B as recordingAccessResponseSchema, D as recordingArtifactSchema, F as recordingModeSchema } from './types-D76SpA3g.js';
+export { h as Asset, g as AssetKind, A as Avatar, o as AvatarClip, c as AvatarSourceSwap, b as AvatarUpdate, p as CallConnection, C as CallMode, a as CallPolicy, q as CallQueued, r as ClipDeclaration, f as ClipLibrary, d as ClipLibraryDeclaration, s as ClipLibraryPlan, e as ClipLibraryUpdate, t as ClipSource, u as ContextMessage, n as CreditBalance, E as EndCallOptions, v as EndCallReason, k as ListRecordingsQuery, l as ListRecordingsResponse, i as ListSessionsOptions, L as LoopRedirect, m as RecordingAccessResponse, R as RecordingArtifact, w as RecordingArtifactStatus, x as RecordingMode, S as StartCallResult, T as TranscriptPayload, j as UsageSession, U as UsageSessionPage, V as VideoPolicy, y as isQueued, z as listRecordingsResponseSchema, B as recordingAccessResponseSchema, D as recordingArtifactSchema, F as recordingModeSchema } from './types-Cmto2uGs.js';
 import './connection-history.js';
 import 'zod';

package/dist/server.js

@@ -1,2 +1,2 @@
-export { RealtimeAvatar, RealtimeAvatarError, RealtimeAvatarHttpError, clipLibraryDeclarationSchema, isQueued, listRecordingsResponseSchema, recordingAccessResponseSchema, recordingArtifactSchema, recordingModeSchema, verifyTranscript } from './chunk-PKOWNCDT.js';
+export { RealtimeAvatar, RealtimeAvatarError, RealtimeAvatarHttpError, clipLibraryDeclarationSchema, isQueued, listRecordingsResponseSchema, recordingAccessResponseSchema, recordingArtifactSchema, recordingModeSchema, verifyTranscript } from './chunk-N76M4MEU.js';
 import './chunk-MCIXU6XP.js';

package/dist/tanstack-start.d.ts

@@ -1,5 +1,5 @@
-import { P as ProxyConfig } from './types-TaQCvMPh.js';
-import './types-D76SpA3g.js';
+import { P as ProxyConfig } from './types-k6pW-wKA.js';
+import './types-Cmto2uGs.js';
 import 'zod';
 
 /**

package/dist/tanstack-start.js

@@ -1,5 +1,5 @@
-import { createProxyHandler } from './chunk-QT5UTARL.js';
-import './chunk-PKOWNCDT.js';
+import { createProxyHandler } from './chunk-T3RZSVRI.js';
+import './chunk-N76M4MEU.js';
 import './chunk-MCIXU6XP.js';
 
 // src/tanstack-start.ts

package/dist/{types-D76SpA3g.d.ts → types-Cmto2uGs.d.ts}

@@ -50,6 +50,12 @@ interface components {
              * @enum {string}
              */
             stt_mode: "server" | "off";
+            /** @description Optional speech recognition hints. Omit language_codes or send [] for automatic multilingual detection. Custom vocabulary biases recognition; at most 100 relevant terms is recommended. It is not a character instruction or a guarantee of recognition accuracy or latency. */
+            transcription?: {
+                language_codes?: string[];
+                custom_vocabulary?: string[];
+            };
+            camera?: boolean;
             video_cache_id?: string | null;
             voice?: ({
                 /** @constant */
@@ -173,9 +179,10 @@ interface components {
             /** @description DEPRECATED, no-op. Accepted and ignored. The faster hesitation ceiling this requested is now the default for every session, so sending it changes nothing. Safe to stop sending; the field is retained only so existing callers do not break. */
             fast_endpointing?: boolean;
             require_ready_clip_library?: boolean;
-            camera?: boolean;
         };
         LiveKitSessionGrant: {
+            /** @description Returned only for X-RTA-Observability: identity-v1 after successful platform billing attachment. The authoritative billing hold tenant ID; never client metadata or provider identity. Omitted by default and on failed or queued responses. Older servers may omit it. */
+            readonly usage_account_id?: string;
             /** @description Returned only for X-RTA-Release-Evidence: 1. Omitted by default for strict SDK compatibility. Missing deployment or worker evidence remains null. */
             readonly releaseIdentity?: {
                 deploymentId: string | null;
@@ -209,6 +216,7 @@ interface components {
              * @enum {string}
              */
             stt_mode: "server" | "off";
+            camera?: boolean;
             /** @default false */
             room_created: boolean;
             /** @default false */
@@ -266,7 +274,6 @@ interface components {
                 /** Format: date-time */
                 expiresAt: string;
             };
-            camera?: boolean;
         };
         LiveKitCapacitySnapshot: {
             capacity_pool: string;
@@ -1165,6 +1172,12 @@ interface CallPolicy {
     /** Speech recognition. `server` for a spoken conversation, `off` if you drive turns. */
     listen?: boolean;
     camera?: Wire["LiveKitSessionRequest"]["camera"];
+    /** Optional language and vocabulary hints for speech recognition. Omit languages for automatic detection. */
+    transcription?: {
+        languageCodes?: NonNullable<Wire["LiveKitSessionRequest"]["transcription"]>["language_codes"];
+        /** Prefer at most 100 relevant names or terms; these are recognition hints, not instructions. */
+        customVocabulary?: NonNullable<Wire["LiveKitSessionRequest"]["transcription"]>["custom_vocabulary"];
+    };
     /** How she is rendered. */
     video?: VideoPolicy;
     /** Voice override for this call; omit to use the avatar's default. */

package/dist/{types-TaQCvMPh.d.ts → types-k6pW-wKA.d.ts}

@@ -1,4 +1,4 @@
-import { C as CallMode, a as CallPolicy } from './types-D76SpA3g.js';
+import { C as CallMode, a as CallPolicy } from './types-Cmto2uGs.js';
 
 /** The three things a caller can ask the proxy to do. Gate on these, not on URLs. */
 type ProxyOperation = "connect" | "end" | "avatars" | "credits";

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "realtime-avatar",
-  "version": "0.20.0",
+  "version": "0.22.0",
   "description": "Realtime Avatar SDK. `realtime-avatar` and the route adapters hold your API key and are server-only; `realtime-avatar/react`, `/browser` and `/tools` never can — importing a server entry into a browser build throws at bundle time.",
   "license": "MIT",
   "author": "The Influence Company",