realtime-avatar 0.25.0 → 0.26.0

package/dist/react.d.ts

@@ -1,10 +1,10 @@
-import { RegisteredTool, ToolRegistration } from './tools.js';
-import { aj as SessionLifecycleRoomBridgeProps, h as AvatarSessionClient, A as AvatarVideoFit } from './proxy-client-O-Mu9JKt.js';
-export { a as AdaptivePlayoutController, b as AdaptivePlayoutDecision, c as AdaptivePlayoutOptions, d as AdaptivePlayoutSample, e as ApproachingEndEvent, f as ApproachingEndReason, g as AvatarConnectionDetails, aQ as AvatarVideoSurface, aR as AvatarVideoSurfaceProps, B as BehaviorSnapshot, C as CallTranscript, i as CallTranscriptSegment, j as CartesiaTtsModel, k as ClipResult, l as ClosingTurnResult, m as CreditsLowEvent, D as DEFAULT_APPROACHING_END_LEAD_SECONDS, n as DEFAULT_CREDITS_LOW_LEAD_SECONDS, o as DEFAULT_GOVERNOR_CONFIG, p as DEFAULT_GRACE_CEILING_SECONDS, q as DEFAULT_GRACE_WINDOW_LEAD_SECONDS, r as DEFAULT_IDLE_SECONDS, s as DEFAULT_IDLE_WARN_LEAD_SECONDS, t as DEFAULT_TURN_TIMEOUT_SECONDS, u as DeclaredInputSource, E as EndReason, v as EndedEvent, w as ExtendResult, F as FishTtsModel, x as FreezeReadingFn, G as Governor, y as GovernorAction, z as GovernorConfig, H as GovernorSignal, I as GovernorState, J as GraceWindowClosedEvent, K as GraceWindowOpenEvent, M as GraceWindowState, N as IdleWarningEvent, O as InboundRtpCursor, P as InboundRtpReading, Q as KnownBehaviorState, R as LLMProvider, S as LLMSelection, T as LiveKitAvatarGrantState, U as LiveKitAvatarGrantStatus, V as LiveKitCapacityState, W as LiveKitConnectionStatus, X as LiveKitSessionRequest, Y as LiveKitSessionStartResult, aS as LivePlaybackKeeper, Z as MAX_RECONNECT_ATTEMPTS, _ as MAX_SESSION_INSTRUCTIONS_CHARS, aT as MIC_LEASE_ENDED_TIMEOUT_MS, aU as MicLease, aV as NETWORK_QUALITY_POLICY, aW as NetworkQualityStatus, aX as PlayableVideoElement, $ as ProxyClientOptions, a0 as QualityCap, a1 as RECONNECT_BACKOFF_MS, aY as RealtimeAvatarCapacityError, aZ as RealtimeAvatarLiveKitRoom, a_ as RealtimeAvatarLiveKitRoomProps, a2 as RealtimeAvatarRequestOptions, a3 as RealtimeSessionApi, a4 as RealtimeSessionMedia, a5 as RealtimeSessionRoomSinks, a6 as ReconnectPolicy, a7 as ReconnectingEvent, a8 as RecoveryState, a9 as RetryStep, aa as SendTurnOptions, ab as SessionBehavior, ac as SessionClip, ad as SessionClocks, ae as SessionEndReason, af as SessionLifecycleApi, ag as SessionLifecyclePhase, ah as SessionLifecyclePhaseKind, ai as SessionLifecycleRoomBridge, ak as SurfaceLayers, al as TranscriptSender, am as TranscriptSenderOptions, an as TurnState, ao as TurnTimeoutEvent, ap as UseAvatarQualityGovernorInput, aq as UseLiveKitAvatarGrantInput, ar as UseRealtimeSessionInput, as as UseSessionLifecycleInput, at as VoiceSpec, au as VoiceSpecInput, av as capacityErrorFromBusy, aw as capacityStateFromGrant, ax as createProxyClient, az as knownBehaviorStates, aA as mapTurnState, aB as readInboundRtp, aC as sessionBehaviorSchema, aD as sessionClipSchema, a$ as shouldReplayPendingTurn, aE as splitCallTranscript, aF as useAvatarAdaptivePlayoutDelay, aG as useAvatarCamera, aH as useAvatarPlayoutDelay, aI as useAvatarQualityGovernor, aJ as useCallTranscript, aK as useLiveKitAvatarGrant, aM as useMicLease, aN as useRealtimeSession, aO as useReleaseMicLeaseOnTrackEnded, aP as useSessionLifecycle } from './proxy-client-O-Mu9JKt.js';
-import { ReactNode } from 'react';
+import { ToolRegistration, RegisteredTool } from './tools.js';
+import { aj as SessionLifecycleRoomBridgeProps, h as AvatarSessionClient, A as AvatarVideoFit, aQ as CallMicrophone, aR as CallAudioPlayback, aS as CallMicrophoneProblem } from './proxy-client-C8M_G_Kk.js';
+export { a as AdaptivePlayoutController, b as AdaptivePlayoutDecision, c as AdaptivePlayoutOptions, d as AdaptivePlayoutSample, e as ApproachingEndEvent, f as ApproachingEndReason, g as AvatarConnectionDetails, aT as AvatarVideoSurface, aU as AvatarVideoSurfaceProps, B as BehaviorSnapshot, C as CallTranscript, i as CallTranscriptSegment, j as CartesiaTtsModel, k as ClipResult, l as ClosingTurnResult, m as CreditsLowEvent, D as DEFAULT_APPROACHING_END_LEAD_SECONDS, n as DEFAULT_CREDITS_LOW_LEAD_SECONDS, o as DEFAULT_GOVERNOR_CONFIG, p as DEFAULT_GRACE_CEILING_SECONDS, q as DEFAULT_GRACE_WINDOW_LEAD_SECONDS, r as DEFAULT_IDLE_SECONDS, s as DEFAULT_IDLE_WARN_LEAD_SECONDS, t as DEFAULT_TURN_TIMEOUT_SECONDS, u as DeclaredInputSource, E as EndReason, v as EndedEvent, w as ExtendResult, F as FishTtsModel, x as FreezeReadingFn, G as Governor, y as GovernorAction, z as GovernorConfig, H as GovernorSignal, I as GovernorState, J as GraceWindowClosedEvent, K as GraceWindowOpenEvent, M as GraceWindowState, N as IdleWarningEvent, O as InboundRtpCursor, P as InboundRtpReading, Q as KnownBehaviorState, R as LLMProvider, S as LLMSelection, T as LiveKitAvatarGrantState, U as LiveKitAvatarGrantStatus, V as LiveKitCapacityState, W as LiveKitConnectionStatus, X as LiveKitSessionRequest, Y as LiveKitSessionStartResult, aV as LivePlaybackKeeper, Z as MAX_RECONNECT_ATTEMPTS, _ as MAX_SESSION_INSTRUCTIONS_CHARS, aW as MIC_LEASE_ENDED_TIMEOUT_MS, aX as MicLease, aY as NETWORK_QUALITY_POLICY, aZ as NetworkQualityStatus, a_ as PlayableVideoElement, $ as ProxyClientOptions, a0 as QualityCap, a1 as RECONNECT_BACKOFF_MS, a$ as RealtimeAvatarCapacityError, b0 as RealtimeAvatarLiveKitRoom, b1 as RealtimeAvatarLiveKitRoomProps, a2 as RealtimeAvatarRequestOptions, a3 as RealtimeSessionApi, a4 as RealtimeSessionMedia, a5 as RealtimeSessionRoomSinks, a6 as ReconnectPolicy, a7 as ReconnectingEvent, a8 as RecoveryState, a9 as RetryStep, aa as SendTurnOptions, ab as SessionBehavior, ac as SessionClip, ad as SessionClocks, ae as SessionEndReason, af as SessionLifecycleApi, ag as SessionLifecyclePhase, ah as SessionLifecyclePhaseKind, ai as SessionLifecycleRoomBridge, ak as SurfaceLayers, al as TranscriptSender, am as TranscriptSenderOptions, an as TurnState, ao as TurnTimeoutEvent, ap as UseAvatarQualityGovernorInput, aq as UseLiveKitAvatarGrantInput, ar as UseRealtimeSessionInput, as as UseSessionLifecycleInput, at as VoiceSpec, au as VoiceSpecInput, av as capacityErrorFromBusy, aw as capacityStateFromGrant, ax as createProxyClient, az as knownBehaviorStates, aA as mapTurnState, aB as readInboundRtp, aC as sessionBehaviorSchema, aD as sessionClipSchema, b2 as shouldReplayPendingTurn, aE as splitCallTranscript, aF as useAvatarAdaptivePlayoutDelay, aG as useAvatarCamera, aH as useAvatarPlayoutDelay, aI as useAvatarQualityGovernor, aJ as useCallTranscript, aK as useLiveKitAvatarGrant, aM as useMicLease, aN as useRealtimeSession, aO as useReleaseMicLeaseOnTrackEnded, aP as useSessionLifecycle } from './proxy-client-C8M_G_Kk.js';
+import { CSSProperties, ReactNode } from 'react';
 import * as react_jsx_runtime from 'react/jsx-runtime';
 import { R as RecordingArtifact } from './recording-DjswTx5B.js';
-export { D as DEFAULT_AVATAR_PLAYOUT_DELAY_SECONDS } from './playout-delay-DWOc8iCl.js';
+export { D as DEFAULT_AVATAR_PLAYOUT_DELAY_SECONDS } from './playout-delay-CtRZdtsj.js';
 import 'zod';
 import 'livekit-client';
 import '@livekit/components-react';
@@ -18,6 +18,22 @@ import '@livekit/components-react';
  */
 declare const DEFAULT_AVATAR_UNSTABLE_STALL_MS = 4000;
 
+/**
+ * Where her tools stand in this call.
+ *
+ * - `idle`: the room is not connected, so there is nothing to register with.
+ * - `registering`: publishing the manifest; `attempt` counts from 1 and rises on each retry.
+ * - `ready`: armed; `registered` is what the agent accepted, `rejected` what it dropped and why.
+ * - `error`: registration failed for good while connected; `error` says why. A tool that is not
+ *   armed is silently uncallable, which reads exactly like the model ignoring it.
+ */
+type CharacterToolsState = {
+    status: "idle" | "registering" | "ready" | "error";
+    registered: string[];
+    rejected: ToolRegistration["rejected"];
+    error?: string;
+    attempt?: number;
+};
 declare function useCharacterTools(tools: Record<string, RegisteredTool>, options?: {
     onResult?: (event: {
         name: string;
@@ -25,12 +41,7 @@ declare function useCharacterTools(tools: Record<string, RegisteredTool>, option
         ok: boolean;
         error?: string;
     }) => void;
-}): {
-    status: "idle" | "registering" | "ready" | "error";
-    registered: string[];
-    rejected: ToolRegistration["rejected"];
-    error?: string;
-};
+}): CharacterToolsState;
 
 /**
  * The CONNECT surface (client half) — one component that puts a live character on screen.
@@ -67,8 +78,34 @@ type AvatarCallStatus =
  | "ended";
 /** Why a call ended — the only vocabulary an end screen needs. */
 type AvatarCallEndReason = "user_ended" | "session_cap" | "idle" | "disconnected" | "out_of_credits" | "agent_ended" | "failed";
+/**
+ * The user's microphone: `off` | `pending` | `on` | `muted`, or `blocked` / `unavailable` with
+ * the cause (`reason`), what the browser said (`message`) and what to do about it (`hint`).
+ */
+type AvatarCallMicrophone = CallMicrophone;
+/** The microphone states that carry a problem — what `onMicrophoneProblem` receives. */
+type AvatarCallMicrophoneProblem = CallMicrophoneProblem;
+/** Whether her audio can play: `unknown` before the room connects, `allowed`, or `blocked`. */
+type AvatarCallAudio = CallAudioPlayback;
 type AvatarCallHandle = {
     status: AvatarCallStatus;
+    /** Can she hear the user? A `blocked` or `unavailable` microphone means she cannot. */
+    microphone: AvatarCallMicrophone;
+    /** Can the user hear her? `blocked` means silent until `startAudio()` runs in a gesture. */
+    audio: AvatarCallAudio;
+    /**
+     * Unblock her audio. Call it from a click or tap handler; outside one the browser refuses.
+     * Never rejects: resolves whether playback is allowed afterwards.
+     */
+    startAudio: () => Promise<boolean>;
+    /**
+     * Mute (`false`) or unmute (`true`) the user's microphone. Never rejects. While waiting or
+     * connecting it records the choice and nothing is captured; a mute made then is kept when the
+     * call goes live. After `end()` it does nothing.
+     */
+    setMicrophoneEnabled: (enabled: boolean) => Promise<void>;
+    /** Ask for the microphone again, after the user fixed what `microphone.hint` names. */
+    retryMicrophone: () => Promise<void>;
     /** Place in line while `status === "waiting"`, else null. */
     queuePosition: number | null;
     /** Seconds until the hard cap, or null before the clock lands. */
@@ -86,7 +123,11 @@ type AvatarCallHandle = {
 };
 type AvatarCallProps = Pick<SessionLifecycleRoomBridgeProps, "onConnectionDetailsChange"> & {
     client: AvatarSessionClient;
-    /** Which character. */
+    /**
+     * Which character. Changing it starts a new call, EXCEPT after `end()`: an ended call stays
+     * ended until you remount `AvatarCall` (a `key` is the usual way), so no prop change can start
+     * a billed call the user hung up on.
+     */
     avatarId: string;
     /** `voice` is audio-only and cheaper; `avatar` (default) is the full call. */
     mode?: "avatar" | "voice";
@@ -97,7 +138,10 @@ type AvatarCallProps = Pick<SessionLifecycleRoomBridgeProps, "onConnectionDetail
     /** A still shown before the idle clip is playable. */
     poster?: string | null;
     fit?: AvatarVideoFit;
+    /** Adds your classes to the call's box. The box sizes itself inline; to resize it, use `style`. */
     className?: string;
+    /** Inline style for the call's box, spread over its own (`height`, `aspectRatio`, …). */
+    style?: CSSProperties;
     /** Your remaining balance in ms, if you want `onLowBalance`. */
     balanceMs?: number;
     onStatusChange?: (status: AvatarCallStatus) => void;
@@ -117,7 +161,24 @@ type AvatarCallProps = Pick<SessionLifecycleRoomBridgeProps, "onConnectionDetail
     onLowBalance?: (event: {
         secondsLeft: number;
     }) => void;
-    /** Overlay your own UI on the video; receives the same handle as `useAvatarCall`. */
+    /**
+     * The microphone became `blocked` or `unavailable`: she cannot hear the user, though the call
+     * is live. Fires once per distinct problem. Show `hint`, then offer `retryMicrophone()`.
+     */
+    onMicrophoneProblem?: (problem: AvatarCallMicrophoneProblem) => void;
+    /**
+     * What to show while `audio` is `blocked` (the browser is muting her until a gesture):
+     * - `true` (default): a "Tap to turn on sound" button, top-centre over the video.
+     * - a function: your own affordance, with your own words; wire it to `call.startAudio()`.
+     * - `false`: nothing; draw your own from `call.audio` anywhere.
+     * The first two render inside a polite live region, above your overlay.
+     */
+    audioUnlockPrompt?: boolean | ((call: AvatarCallHandle) => ReactNode);
+    /**
+     * Overlay your own UI on the video; receives the same handle as `useAvatarCall`. Rendered in a
+     * layer above the video and the live badge that fills the call's box, so `position: absolute`
+     * places a control against the box and it stays clickable while she is live.
+     */
     children?: (call: AvatarCallHandle) => ReactNode;
 };
 /**
@@ -141,4 +202,4 @@ declare function RecordingPlayer({ assets, className }: {
     className?: string;
 }): react_jsx_runtime.JSX.Element;
 
-export { AvatarCall, type AvatarCallEndReason, type AvatarCallHandle, type AvatarCallProps, type AvatarCallStatus, AvatarSessionClient, AvatarVideoFit, DEFAULT_AVATAR_UNSTABLE_STALL_MS, type RecordingPlaybackAsset, RecordingPlayer, SessionLifecycleRoomBridgeProps, useAvatarCall, useCharacterTools };
+export { AvatarCall, type AvatarCallAudio, type AvatarCallEndReason, type AvatarCallHandle, type AvatarCallMicrophone, type AvatarCallMicrophoneProblem, type AvatarCallProps, type AvatarCallStatus, AvatarSessionClient, AvatarVideoFit, type CharacterToolsState, DEFAULT_AVATAR_UNSTABLE_STALL_MS, type RecordingPlaybackAsset, RecordingPlayer, SessionLifecycleRoomBridgeProps, useAvatarCall, useCharacterTools };