realtime-avatar 0.9.1 → 0.11.0

package/dist/react-native.js

@@ -858,38 +858,74 @@ var qualityToSignal = (q) => {
   }
 };
 function useAvatarQualityGovernor(input) {
-  const { enabled, freezeReading, config = DEFAULT_GOVERNOR_CONFIG, tickMs = 1e3 } = input;
+  const { enabled, freezeReading, config: policy = DEFAULT_GOVERNOR_CONFIG, tickMs = 1e3 } = input;
   const room = useMaybeRoomContext();
   const { videoTrack } = useVoiceAssistant();
-  const pausedSinceTick = useRef(false);
-  const connQuality = useRef("unknown");
-  const lastFreezeStat = useRef(null);
-  const jitterTrend = useRef(null);
+  const {
+    openingCap,
+    downgradeFreezeMs,
+    probationFreezeMs,
+    openingDwellMs,
+    dwellBaseMs,
+    dwellMaxMs,
+    cleanMs,
+    probeMs,
+    healthyResetMs
+  } = policy;
+  const config = useMemo(() => ({
+    openingCap,
+    downgradeFreezeMs,
+    probationFreezeMs,
+    openingDwellMs,
+    dwellBaseMs,
+    dwellMaxMs,
+    cleanMs,
+    probeMs,
+    healthyResetMs
+  }), [
+    openingCap,
+    downgradeFreezeMs,
+    probationFreezeMs,
+    openingDwellMs,
+    dwellBaseMs,
+    dwellMaxMs,
+    cleanMs,
+    probeMs,
+    healthyResetMs
+  ]);
+  const freezeReadingRef = useRef(freezeReading);
+  freezeReadingRef.current = freezeReading;
+  const targetPublication = videoTrack?.publication;
+  const targetParticipant = videoTrack?.participant;
+  const targetTrack = targetPublication?.track;
   useEffect(() => {
-    if (!enabled || !room) return;
-    const targetPublication = videoTrack?.publication;
-    const targetParticipant = videoTrack?.participant;
-    pausedSinceTick.current = false;
-    lastFreezeStat.current = null;
-    jitterTrend.current = null;
-    connQuality.current = qualityToSignal(
+    if (!enabled || !room || !targetPublication || !targetParticipant) return;
+    let pausedSinceTick = false;
+    let lastFreezeStat = null;
+    let jitterTrend = null;
+    let connQuality = qualityToSignal(
       targetParticipant?.connectionQuality ?? ConnectionQuality.Unknown
     );
     const onStreamState = (pub, streamState) => {
-      if (targetPublication && pub !== targetPublication) return;
-      if (streamState === Track.StreamState.Paused) pausedSinceTick.current = true;
+      if (pub !== targetPublication) return;
+      if (streamState === Track.StreamState.Paused) pausedSinceTick = true;
     };
     const onQuality = (q, participant) => {
-      if (targetParticipant && participant.sid !== targetParticipant.sid) return;
-      connQuality.current = qualityToSignal(q);
+      if (participant.sid !== targetParticipant.sid) return;
+      connQuality = qualityToSignal(q);
     };
     try {
       room.on(RoomEvent.TrackStreamStateChanged, onStreamState);
       room.on(RoomEvent.ConnectionQualityChanged, onQuality);
     } catch {
+      try {
+        room.off(RoomEvent.TrackStreamStateChanged, onStreamState);
+        room.off(RoomEvent.ConnectionQualityChanged, onQuality);
+      } catch {
+      }
       return;
     }
-    let gov = initGovernor(Date.now());
+    let gov = initGovernor(Date.now(), config.openingCap);
     const readGetStatsSignals = async () => {
       try {
         const track = targetPublication?.track;
@@ -908,13 +944,13 @@ function useAvatarQualityGovernor(input) {
           }
         });
         const now = Date.now();
-        const prev = lastFreezeStat.current;
-        lastFreezeStat.current = { frozen: frozenTotalMs, ts: now };
-        const trend = stepJitterBufferTrend(jitterTrend.current, {
+        const prev = lastFreezeStat;
+        lastFreezeStat = { frozen: frozenTotalMs, ts: now };
+        const trend = stepJitterBufferTrend(jitterTrend, {
           delaySeconds: jitterDelaySeconds,
           emittedCount: jitterEmittedCount
         });
-        jitterTrend.current = trend.state;
+        jitterTrend = trend.state;
         return {
           freezeMs: prev ? Math.max(0, frozenTotalMs - prev.frozen) : 0,
           jitterRising: trend.rising
@@ -938,17 +974,17 @@ function useAvatarQualityGovernor(input) {
       if (disposed || tickRunning) return;
       tickRunning = true;
       try {
-        const rvfc = freezeReading?.() ?? { freezeMsInWindow: 0, inhibited: false };
+        const rvfc = freezeReadingRef.current?.() ?? { freezeMsInWindow: 0, inhibited: false };
         const statsSignals = await readGetStatsSignals();
         if (disposed) return;
         const signal = {
-          paused: pausedSinceTick.current,
+          paused: pausedSinceTick,
           freezeMsInWindow: Math.max(rvfc.freezeMsInWindow, statsSignals.freezeMs),
           jitterRising: statsSignals.jitterRising,
-          connectionQuality: connQuality.current,
+          connectionQuality: connQuality,
           inhibited: rvfc.inhibited
         };
-        pausedSinceTick.current = false;
+        pausedSinceTick = false;
         const { governor, action } = step(gov, signal, Date.now(), config);
         gov = governor;
         if (action) applyCap(action.setCap);
@@ -968,7 +1004,7 @@ function useAvatarQualityGovernor(input) {
       } catch {
       }
     };
-  }, [enabled, room, videoTrack, freezeReading, config, tickMs]);
+  }, [enabled, room, targetPublication, targetParticipant, targetTrack, config, tickMs]);
 }
 
 // ../client/src/react/avatar-video-surface.ts
@@ -1297,8 +1333,7 @@ var sessionChoreographySchema = z.object({
   { message: "idle_dwell_min_seconds must be <= idle_dwell_max_seconds" }
 );
 var sessionBehaviorSchema = z.object({
-  gestures_enabled: z.boolean().optional(),
-  gesture_freq: z.enum(["sparse", "balanced", "lively"]).optional()
+  gestures_enabled: z.boolean().optional()
 }).strict();
 var transcriptWebhookSchema = z.object({
   url: z.string().url().max(500),
@@ -2398,11 +2433,11 @@ function useRealtimeSession(input) {
     requestGracefulClose();
     lifecycle.reset();
   }, [lifecycle, requestGracefulClose]);
-  const performClip = useCallback(
-    (clipId, opts) => {
+  const performAction = useCallback(
+    (actionId, opts) => {
       const requestId = newTurnId();
       const publish = dataPublisherRef.current;
-      const trimmed = clipId.trim();
+      const trimmed = actionId.trim();
       if (!publish || !trimmed) {
         const result = { requestId, accepted: false, reason: "not_connected" };
         cbRef.current.onClipResult?.(result);
@@ -2417,10 +2452,9 @@ function useRealtimeSession(input) {
         }, opts?.timeoutMs ?? 5e3);
         pendingClipsRef.current.set(requestId, { resolve, timer });
         publish({
-          kind: "clip_request",
+          kind: "action_request",
           request_id: requestId,
-          clip_id: trimmed,
-          ...typeof opts?.holdSeconds === "number" ? { hold_seconds: opts.holdSeconds } : {}
+          action_id: trimmed
         });
       });
     },
@@ -2465,7 +2499,7 @@ function useRealtimeSession(input) {
       retryTurn,
       end,
       behavior,
-      performClip,
+      performAction,
       onLifecycleData,
       registerDataPublisher,
       registerTurnSender,
@@ -2487,7 +2521,7 @@ function useRealtimeSession(input) {
       retryTurn,
       end,
       behavior,
-      performClip,
+      performAction,
       onLifecycleData,
       registerDataPublisher,
       registerTurnSender,

package/dist/react.d.ts

@@ -1,10 +1,25 @@
+import { RegisteredTool, ToolRegistration } from './tools.js';
 import { ReactNode } from 'react';
-import { g as AvatarSessionClient, A as AvatarVideoFit } from './proxy-client-DlYBC6cp.js';
-export { a as AdaptivePlayoutController, b as AdaptivePlayoutDecision, c as AdaptivePlayoutOptions, d as AdaptivePlayoutSample, e as ApproachingEndEvent, f as ApproachingEndReason, aK as AvatarVideoSurface, aL as AvatarVideoSurfaceProps, B as BehaviorSnapshot, C as CallTranscript, h as CallTranscriptSegment, i as CartesiaTtsModel, j as ClipResult, k as ClosingTurnResult, l as CreditsLowEvent, D as DEFAULT_APPROACHING_END_LEAD_SECONDS, m as DEFAULT_CREDITS_LOW_LEAD_SECONDS, n as DEFAULT_GOVERNOR_CONFIG, o as DEFAULT_GRACE_CEILING_SECONDS, p as DEFAULT_GRACE_WINDOW_LEAD_SECONDS, q as DEFAULT_IDLE_SECONDS, r as DEFAULT_IDLE_WARN_LEAD_SECONDS, s as DEFAULT_TURN_TIMEOUT_SECONDS, E as EndReason, t as EndedEvent, u as ExtendResult, F as FishTtsModel, v as FreezeReadingFn, G as Governor, w as GovernorAction, x as GovernorConfig, y as GovernorSignal, z as GovernorState, H as GraceWindowClosedEvent, I as GraceWindowOpenEvent, J as GraceWindowState, K as IdleWarningEvent, M as InboundRtpCursor, N as InboundRtpReading, O as KnownBehaviorState, P as LLMProvider, Q as LLMSelection, R as LiveKitAvatarGrantState, S as LiveKitAvatarGrantStatus, T as LiveKitCapacityState, U as LiveKitConnectionStatus, V as LiveKitSessionRequest, W as LiveKitSessionStartResult, aM as LivePlaybackKeeper, X as MAX_RECONNECT_ATTEMPTS, Y as MAX_SESSION_INSTRUCTIONS_CHARS, aN as MIC_LEASE_ENDED_TIMEOUT_MS, aO as MicLease, aP as PlayableVideoElement, Z as ProxyClientOptions, _ as QualityCap, $ as RECONNECT_BACKOFF_MS, aQ as RealtimeAvatarCapacityError, aR as RealtimeAvatarLiveKitRoom, aS as RealtimeAvatarLiveKitRoomProps, a0 as RealtimeAvatarRequestOptions, a1 as RealtimeSessionApi, a2 as RealtimeSessionMedia, a3 as RealtimeSessionRoomSinks, a4 as ReconnectPolicy, a5 as ReconnectingEvent, a6 as RecoveryState, a7 as RetryStep, a8 as SessionBehavior, a9 as SessionClip, aa as SessionClocks, ab as SessionEndReason, ac as SessionLifecycleApi, ad as SessionLifecyclePhase, ae as SessionLifecyclePhaseKind, af as SessionLifecycleRoomBridge, ag as SessionLifecycleRoomBridgeProps, ah as SurfaceLayers, ai as TurnState, aj as TurnTimeoutEvent, ak as UseAvatarQualityGovernorInput, al as UseLiveKitAvatarGrantInput, am as UseRealtimeSessionInput, an as UseSessionLifecycleInput, ao as VoiceSpec, ap as VoiceSpecInput, aq as capacityErrorFromBusy, ar as capacityStateFromGrant, as as createProxyClient, au as knownBehaviorStates, av as mapTurnState, aw as readInboundRtp, ax as sessionBehaviorSchema, ay as sessionClipSchema, aT as shouldReplayPendingTurn, az as splitCallTranscript, aA as useAvatarAdaptivePlayoutDelay, aB as useAvatarPlayoutDelay, aC as useAvatarQualityGovernor, aD as useCallTranscript, aE as useLiveKitAvatarGrant, aG as useMicLease, aH as useRealtimeSession, aI as useReleaseMicLeaseOnTrackEnded, aJ as useSessionLifecycle } from './proxy-client-DlYBC6cp.js';
+import { g as AvatarSessionClient, A as AvatarVideoFit } from './proxy-client-CWvT16wz.js';
+export { a as AdaptivePlayoutController, b as AdaptivePlayoutDecision, c as AdaptivePlayoutOptions, d as AdaptivePlayoutSample, e as ApproachingEndEvent, f as ApproachingEndReason, aK as AvatarVideoSurface, aL as AvatarVideoSurfaceProps, B as BehaviorSnapshot, C as CallTranscript, h as CallTranscriptSegment, i as CartesiaTtsModel, j as ClipResult, k as ClosingTurnResult, l as CreditsLowEvent, D as DEFAULT_APPROACHING_END_LEAD_SECONDS, m as DEFAULT_CREDITS_LOW_LEAD_SECONDS, n as DEFAULT_GOVERNOR_CONFIG, o as DEFAULT_GRACE_CEILING_SECONDS, p as DEFAULT_GRACE_WINDOW_LEAD_SECONDS, q as DEFAULT_IDLE_SECONDS, r as DEFAULT_IDLE_WARN_LEAD_SECONDS, s as DEFAULT_TURN_TIMEOUT_SECONDS, E as EndReason, t as EndedEvent, u as ExtendResult, F as FishTtsModel, v as FreezeReadingFn, G as Governor, w as GovernorAction, x as GovernorConfig, y as GovernorSignal, z as GovernorState, H as GraceWindowClosedEvent, I as GraceWindowOpenEvent, J as GraceWindowState, K as IdleWarningEvent, M as InboundRtpCursor, N as InboundRtpReading, O as KnownBehaviorState, P as LLMProvider, Q as LLMSelection, R as LiveKitAvatarGrantState, S as LiveKitAvatarGrantStatus, T as LiveKitCapacityState, U as LiveKitConnectionStatus, V as LiveKitSessionRequest, W as LiveKitSessionStartResult, aM as LivePlaybackKeeper, X as MAX_RECONNECT_ATTEMPTS, Y as MAX_SESSION_INSTRUCTIONS_CHARS, aN as MIC_LEASE_ENDED_TIMEOUT_MS, aO as MicLease, aP as PlayableVideoElement, Z as ProxyClientOptions, _ as QualityCap, $ as RECONNECT_BACKOFF_MS, aQ as RealtimeAvatarCapacityError, aR as RealtimeAvatarLiveKitRoom, aS as RealtimeAvatarLiveKitRoomProps, a0 as RealtimeAvatarRequestOptions, a1 as RealtimeSessionApi, a2 as RealtimeSessionMedia, a3 as RealtimeSessionRoomSinks, a4 as ReconnectPolicy, a5 as ReconnectingEvent, a6 as RecoveryState, a7 as RetryStep, a8 as SessionBehavior, a9 as SessionClip, aa as SessionClocks, ab as SessionEndReason, ac as SessionLifecycleApi, ad as SessionLifecyclePhase, ae as SessionLifecyclePhaseKind, af as SessionLifecycleRoomBridge, ag as SessionLifecycleRoomBridgeProps, ah as SurfaceLayers, ai as TurnState, aj as TurnTimeoutEvent, ak as UseAvatarQualityGovernorInput, al as UseLiveKitAvatarGrantInput, am as UseRealtimeSessionInput, an as UseSessionLifecycleInput, ao as VoiceSpec, ap as VoiceSpecInput, aq as capacityErrorFromBusy, ar as capacityStateFromGrant, as as createProxyClient, au as knownBehaviorStates, av as mapTurnState, aw as readInboundRtp, ax as sessionBehaviorSchema, ay as sessionClipSchema, aT as shouldReplayPendingTurn, az as splitCallTranscript, aA as useAvatarAdaptivePlayoutDelay, aB as useAvatarPlayoutDelay, aC as useAvatarQualityGovernor, aD as useCallTranscript, aE as useLiveKitAvatarGrant, aG as useMicLease, aH as useRealtimeSession, aI as useReleaseMicLeaseOnTrackEnded, aJ as useSessionLifecycle } from './proxy-client-CWvT16wz.js';
 export { D as DEFAULT_AVATAR_PLAYOUT_DELAY_SECONDS } from './playout-delay-DWOc8iCl.js';
+import 'zod';
 import '@livekit/components-react';
 import 'livekit-client';
-import 'zod';
+
+declare function useCharacterTools(tools: Record<string, RegisteredTool>, options?: {
+    onResult?: (event: {
+        name: string;
+        callId: string;
+        ok: boolean;
+        error?: string;
+    }) => void;
+}): {
+    status: "idle" | "registering" | "ready" | "error";
+    registered: string[];
+    rejected: ToolRegistration["rejected"];
+    error?: string;
+};
 
 /**
  * The CONNECT surface (client half) — one component that puts a live character on screen.
@@ -105,4 +120,4 @@ declare function useAvatarCall(props: AvatarCallProps): {
 /** One component: a live character on screen. */
 declare function AvatarCall(props: AvatarCallProps): ReactNode;
 
-export { AvatarCall, type AvatarCallEndReason, type AvatarCallHandle, type AvatarCallProps, type AvatarCallStatus, AvatarSessionClient, AvatarVideoFit, useAvatarCall };
+export { AvatarCall, type AvatarCallEndReason, type AvatarCallHandle, type AvatarCallProps, type AvatarCallStatus, AvatarSessionClient, AvatarVideoFit, useAvatarCall, useCharacterTools };

package/dist/react.js

@@ -1,10 +1,204 @@
 "use client";
 import { useRef, useState, useEffect, useCallback, useMemo, createElement, Fragment } from 'react';
-import { useTranscriptions, useLocalParticipant, RoomAudioRenderer, LiveKitRoom, useMaybeRoomContext, useVoiceAssistant, useConnectionState, VideoTrack, useRoomContext, useChat } from '@livekit/components-react';
-import { ConnectionQuality, RoomEvent, ConnectionState, TrackEvent, Track, VideoQuality, ConnectionError, ConnectionErrorReason, DisconnectReason, Room } from 'livekit-client';
+import { useRoomContext, useConnectionState, useTranscriptions, useLocalParticipant, RoomAudioRenderer, LiveKitRoom, useMaybeRoomContext, useVoiceAssistant, VideoTrack, useChat } from '@livekit/components-react';
+import { ConnectionState, ConnectionQuality, RoomEvent, TrackEvent, Track, VideoQuality, ConnectionError, ConnectionErrorReason, DisconnectReason, Room } from 'livekit-client';
 import { z } from 'zod';
 
-// ../client/src/react/avatar-call.ts
+// ../client/src/react/use-character-tools.ts
+var invocationSchema = z.object({
+  name: z.string().regex(/^[a-zA-Z0-9_-]{1,64}$/),
+  args: z.string().max(8192),
+  call_id: z.string().min(1).max(128)
+}).strict();
+var registrationSchema = z.object({
+  accepted: z.array(z.string()),
+  rejected: z.array(z.object({ name: z.string(), reason: z.string() })).default([])
+});
+var MAX_TOOLS = 32;
+var MAX_MANIFEST_BYTES = 12e3;
+var MAX_RESULT_CHARS = 8192;
+var TOOL_DEADLINE_MS = 2500;
+var REGISTER_METHOD = "rta.tools.register";
+var INVOKE_METHOD = "rta.tool.invoke";
+var NAME_RE = /^[a-zA-Z0-9_-]{1,64}$/;
+function candidateIdentities(room) {
+  const all = Array.from(room.remoteParticipants.values(), (p) => p.identity).filter(Boolean);
+  const agents = all.filter((id) => /^agent[-_]/i.test(id));
+  return [...agents, ...all.filter((id) => !agents.includes(id))];
+}
+function methodMissing(error) {
+  const code = error?.code;
+  if (code === "UNSUPPORTED_METHOD" || code === "RECIPIENT_NOT_FOUND" || code === 1401 || code === 1402) {
+    return true;
+  }
+  const message = error instanceof Error ? error.message : String(error ?? "");
+  return /not supported|unsupported_method|recipient_not_found|recipient not found/i.test(message);
+}
+var sleep = (ms) => new Promise((resolve) => setTimeout(resolve, ms));
+function buildManifest(tools) {
+  const descriptors = [];
+  const rejected = [];
+  for (const [name, tool] of Object.entries(tools)) {
+    if (!NAME_RE.test(name)) {
+      rejected.push({ name, reason: "name must match [a-zA-Z0-9_-]{1,64}" });
+      continue;
+    }
+    if (!tool?.description) {
+      rejected.push({ name, reason: "description is required \u2014 it is how she knows when to call it" });
+      continue;
+    }
+    if (typeof tool.execute !== "function") {
+      rejected.push({ name, reason: "execute must be a function" });
+      continue;
+    }
+    if (descriptors.length >= MAX_TOOLS) {
+      rejected.push({ name, reason: `over the ${MAX_TOOLS}-tool limit for one session` });
+      continue;
+    }
+    descriptors.push({ name, description: tool.description, parameters: tool.parameters ?? {} });
+  }
+  return { descriptors, rejected };
+}
+async function attachAvatarTools(room, tools, options = {}) {
+  const { descriptors, rejected } = buildManifest(tools);
+  const payload = JSON.stringify({ tools: descriptors });
+  if (payload.length > MAX_MANIFEST_BYTES) {
+    throw new Error(
+      `tool manifest is ${payload.length} bytes, over the ${MAX_MANIFEST_BYTES} limit \u2014 shorten descriptions or register fewer tools`
+    );
+  }
+  options.signal?.throwIfAborted();
+  let disposed = false;
+  let activeCaller;
+  const inFlight = /* @__PURE__ */ new Set();
+  const dispose = () => {
+    if (disposed) return;
+    disposed = true;
+    room.localParticipant.unregisterRpcMethod?.(INVOKE_METHOD);
+    for (const call of inFlight) call.abort();
+    options.signal?.removeEventListener("abort", dispose);
+  };
+  options.signal?.addEventListener("abort", dispose, { once: true });
+  room.localParticipant.registerRpcMethod(INVOKE_METHOD, async ({ payload: payload2, callerIdentity }) => {
+    let name = "", callId = "";
+    try {
+      if (disposed) throw new Error("Tool registration was disposed");
+      if (callerIdentity && callerIdentity !== activeCaller) throw new Error("Unregistered tool caller");
+      const req = invocationSchema.parse(JSON.parse(payload2));
+      name = req.name;
+      callId = req.call_id;
+      const tool = tools[name];
+      if (!tool) return JSON.stringify({ ok: false, call_id: callId, error: `no tool named ${name}` });
+      const args = z.record(z.string(), z.unknown()).parse(req.args ? JSON.parse(req.args) : {});
+      options.onInvoke?.(name, args);
+      const run = tool.execute;
+      const controller = new AbortController();
+      inFlight.add(controller);
+      const timer = setTimeout(() => controller.abort(), TOOL_DEADLINE_MS);
+      let result;
+      try {
+        result = await run(args, { signal: controller.signal, callId });
+        controller.signal.throwIfAborted();
+      } finally {
+        inFlight.delete(controller);
+        clearTimeout(timer);
+      }
+      const encoded = typeof result === "string" ? result : JSON.stringify(result ?? null);
+      if (encoded.length > MAX_RESULT_CHARS) {
+        return JSON.stringify({
+          ok: false,
+          call_id: callId,
+          error: `result was ${encoded.length} chars, over the ${MAX_RESULT_CHARS} limit`
+        });
+      }
+      options.onResult?.({ name, callId, ok: true });
+      return JSON.stringify({ ok: true, call_id: callId, result: encoded });
+    } catch (error) {
+      options.onResult?.({ name, callId, ok: false, error: error instanceof Error ? error.message : "tool failed" });
+      return JSON.stringify({
+        ok: false,
+        call_id: callId,
+        error: error instanceof Error ? error.message : "tool failed"
+      });
+    }
+  });
+  const deadline = Date.now() + Math.max(0, options.timeoutMs ?? 8e3);
+  let lastError = null;
+  let sawCandidate = false;
+  try {
+    for (; ; ) {
+      options.signal?.throwIfAborted();
+      for (const identity of candidateIdentities(room)) {
+        sawCandidate = true;
+        activeCaller = identity;
+        try {
+          const raw = await room.localParticipant.performRpc({
+            destinationIdentity: identity,
+            method: REGISTER_METHOD,
+            payload
+          });
+          options.signal?.throwIfAborted();
+          const result = registrationSchema.parse(JSON.parse(raw));
+          return {
+            accepted: result.accepted,
+            rejected: [...rejected, ...result.rejected],
+            dispose
+          };
+        } catch (error) {
+          activeCaller = void 0;
+          lastError = error;
+          if (!methodMissing(error)) throw error;
+        }
+      }
+      if (Date.now() >= deadline) break;
+      await sleep(200);
+    }
+    throw new Error(
+      sawCandidate ? `no participant accepted the tool manifest \u2014 either this session was not minted with the \`client_tools\` capability (a server-side grant), or the agent never armed registration within ${options.timeoutMs ?? 8e3}ms` + (lastError instanceof Error ? ` (last error: ${lastError.message})` : "") : "no remote participant joined the room before the timeout \u2014 nothing to register with"
+    );
+  } catch (error) {
+    dispose();
+    throw error;
+  }
+}
+
+// ../client/src/react/use-character-tools.ts
+function useCharacterTools(tools, options = {}) {
+  const room = useRoomContext();
+  const connection = useConnectionState();
+  const callback = useRef(options.onResult);
+  callback.current = options.onResult;
+  const [state, setState] = useState({ status: "idle", registered: [], rejected: [] });
+  useEffect(() => {
+    if (connection !== ConnectionState.Connected) {
+      setState({ status: "idle", registered: [], rejected: [] });
+      return;
+    }
+    const abort = new AbortController();
+    setState({ status: "registering", registered: [], rejected: [] });
+    void attachAvatarTools(room, tools, {
+      signal: abort.signal,
+      onResult: (event) => {
+        if (!abort.signal.aborted) callback.current?.(event);
+      }
+    }).then((result) => {
+      if (abort.signal.aborted) {
+        result.dispose();
+        return;
+      }
+      setState({ status: "ready", registered: result.accepted, rejected: result.rejected });
+    }).catch((error) => {
+      if (!abort.signal.aborted) setState({
+        status: "error",
+        registered: [],
+        rejected: [],
+        error: error instanceof Error ? error.message : "Tool registration failed"
+      });
+    });
+    return () => abort.abort();
+  }, [room, tools, connection]);
+  return state;
+}
 var holder = null;
 var queue = [];
 var pendingEnded = null;
@@ -865,38 +1059,74 @@ var qualityToSignal = (q) => {
   }
 };
 function useAvatarQualityGovernor(input) {
-  const { enabled, freezeReading, config = DEFAULT_GOVERNOR_CONFIG, tickMs = 1e3 } = input;
+  const { enabled, freezeReading, config: policy = DEFAULT_GOVERNOR_CONFIG, tickMs = 1e3 } = input;
   const room = useMaybeRoomContext();
   const { videoTrack } = useVoiceAssistant();
-  const pausedSinceTick = useRef(false);
-  const connQuality = useRef("unknown");
-  const lastFreezeStat = useRef(null);
-  const jitterTrend = useRef(null);
+  const {
+    openingCap,
+    downgradeFreezeMs,
+    probationFreezeMs,
+    openingDwellMs,
+    dwellBaseMs,
+    dwellMaxMs,
+    cleanMs,
+    probeMs,
+    healthyResetMs
+  } = policy;
+  const config = useMemo(() => ({
+    openingCap,
+    downgradeFreezeMs,
+    probationFreezeMs,
+    openingDwellMs,
+    dwellBaseMs,
+    dwellMaxMs,
+    cleanMs,
+    probeMs,
+    healthyResetMs
+  }), [
+    openingCap,
+    downgradeFreezeMs,
+    probationFreezeMs,
+    openingDwellMs,
+    dwellBaseMs,
+    dwellMaxMs,
+    cleanMs,
+    probeMs,
+    healthyResetMs
+  ]);
+  const freezeReadingRef = useRef(freezeReading);
+  freezeReadingRef.current = freezeReading;
+  const targetPublication = videoTrack?.publication;
+  const targetParticipant = videoTrack?.participant;
+  const targetTrack = targetPublication?.track;
   useEffect(() => {
-    if (!enabled || !room) return;
-    const targetPublication = videoTrack?.publication;
-    const targetParticipant = videoTrack?.participant;
-    pausedSinceTick.current = false;
-    lastFreezeStat.current = null;
-    jitterTrend.current = null;
-    connQuality.current = qualityToSignal(
+    if (!enabled || !room || !targetPublication || !targetParticipant) return;
+    let pausedSinceTick = false;
+    let lastFreezeStat = null;
+    let jitterTrend = null;
+    let connQuality = qualityToSignal(
       targetParticipant?.connectionQuality ?? ConnectionQuality.Unknown
     );
     const onStreamState = (pub, streamState) => {
-      if (targetPublication && pub !== targetPublication) return;
-      if (streamState === Track.StreamState.Paused) pausedSinceTick.current = true;
+      if (pub !== targetPublication) return;
+      if (streamState === Track.StreamState.Paused) pausedSinceTick = true;
     };
     const onQuality = (q, participant) => {
-      if (targetParticipant && participant.sid !== targetParticipant.sid) return;
-      connQuality.current = qualityToSignal(q);
+      if (participant.sid !== targetParticipant.sid) return;
+      connQuality = qualityToSignal(q);
     };
     try {
       room.on(RoomEvent.TrackStreamStateChanged, onStreamState);
       room.on(RoomEvent.ConnectionQualityChanged, onQuality);
     } catch {
+      try {
+        room.off(RoomEvent.TrackStreamStateChanged, onStreamState);
+        room.off(RoomEvent.ConnectionQualityChanged, onQuality);
+      } catch {
+      }
       return;
     }
-    let gov = initGovernor(Date.now());
+    let gov = initGovernor(Date.now(), config.openingCap);
     const readGetStatsSignals = async () => {
       try {
         const track = targetPublication?.track;
@@ -915,13 +1145,13 @@ function useAvatarQualityGovernor(input) {
           }
         });
         const now = Date.now();
-        const prev = lastFreezeStat.current;
-        lastFreezeStat.current = { frozen: frozenTotalMs, ts: now };
-        const trend = stepJitterBufferTrend(jitterTrend.current, {
+        const prev = lastFreezeStat;
+        lastFreezeStat = { frozen: frozenTotalMs, ts: now };
+        const trend = stepJitterBufferTrend(jitterTrend, {
           delaySeconds: jitterDelaySeconds,
           emittedCount: jitterEmittedCount
         });
-        jitterTrend.current = trend.state;
+        jitterTrend = trend.state;
         return {
           freezeMs: prev ? Math.max(0, frozenTotalMs - prev.frozen) : 0,
           jitterRising: trend.rising
@@ -945,17 +1175,17 @@ function useAvatarQualityGovernor(input) {
       if (disposed || tickRunning) return;
       tickRunning = true;
       try {
-        const rvfc = freezeReading?.() ?? { freezeMsInWindow: 0, inhibited: false };
+        const rvfc = freezeReadingRef.current?.() ?? { freezeMsInWindow: 0, inhibited: false };
         const statsSignals = await readGetStatsSignals();
         if (disposed) return;
         const signal = {
-          paused: pausedSinceTick.current,
+          paused: pausedSinceTick,
           freezeMsInWindow: Math.max(rvfc.freezeMsInWindow, statsSignals.freezeMs),
           jitterRising: statsSignals.jitterRising,
-          connectionQuality: connQuality.current,
+          connectionQuality: connQuality,
           inhibited: rvfc.inhibited
         };
-        pausedSinceTick.current = false;
+        pausedSinceTick = false;
         const { governor, action } = step(gov, signal, Date.now(), config);
         gov = governor;
         if (action) applyCap(action.setCap);
@@ -975,7 +1205,7 @@ function useAvatarQualityGovernor(input) {
       } catch {
       }
     };
-  }, [enabled, room, videoTrack, freezeReading, config, tickMs]);
+  }, [enabled, room, targetPublication, targetParticipant, targetTrack, config, tickMs]);
 }
 
 // ../client/src/react/frame-recovery.ts
@@ -1027,6 +1257,8 @@ function AvatarVideoSurface(props) {
     onPlayoutDelayChange
   } = props;
   const testId = props["data-testid"];
+  const visibilityCallback = useRef(props.onLiveVisibilityChange);
+  visibilityCallback.current = props.onLiveVisibilityChange;
   const { videoTrack, audioTrack } = useVoiceAssistant();
   useAvatarPlayoutDelay(videoTrack, audioTrack);
   const playoutDelaySeconds = useAvatarAdaptivePlayoutDelay(
@@ -1069,6 +1301,9 @@ function AvatarVideoSurface(props) {
     liveWanted,
     connected && !networkStalled ? idleReturnDelayMs : 0
   );
+  useEffect(() => {
+    visibilityCallback.current?.(showLive);
+  }, [showLive]);
   const liveDims = videoTrack?.publication?.dimensions ?? null;
   const idleVideoRef = useRef(null);
   const boxAspect = aspectRatio ?? null;
@@ -1572,8 +1807,7 @@ var sessionChoreographySchema = z.object({
   { message: "idle_dwell_min_seconds must be <= idle_dwell_max_seconds" }
 );
 var sessionBehaviorSchema = z.object({
-  gestures_enabled: z.boolean().optional(),
-  gesture_freq: z.enum(["sparse", "balanced", "lively"]).optional()
+  gestures_enabled: z.boolean().optional()
 }).strict();
 var transcriptWebhookSchema = z.object({
   url: z.string().url().max(500),
@@ -2678,11 +2912,11 @@ function useRealtimeSession(input) {
     requestGracefulClose();
     lifecycle.reset();
   }, [lifecycle, requestGracefulClose]);
-  const performClip = useCallback(
-    (clipId, opts) => {
+  const performAction = useCallback(
+    (actionId, opts) => {
       const requestId = newTurnId();
       const publish = dataPublisherRef.current;
-      const trimmed = clipId.trim();
+      const trimmed = actionId.trim();
       if (!publish || !trimmed) {
         const result = { requestId, accepted: false, reason: "not_connected" };
         cbRef.current.onClipResult?.(result);
@@ -2697,10 +2931,9 @@ function useRealtimeSession(input) {
         }, opts?.timeoutMs ?? 5e3);
         pendingClipsRef.current.set(requestId, { resolve, timer });
         publish({
-          kind: "clip_request",
+          kind: "action_request",
           request_id: requestId,
-          clip_id: trimmed,
-          ...typeof opts?.holdSeconds === "number" ? { hold_seconds: opts.holdSeconds } : {}
+          action_id: trimmed
         });
       });
     },
@@ -2745,7 +2978,7 @@ function useRealtimeSession(input) {
       retryTurn,
       end,
       behavior,
-      performClip,
+      performAction,
       onLifecycleData,
       registerDataPublisher,
       registerTurnSender,
@@ -2767,7 +3000,7 @@ function useRealtimeSession(input) {
       retryTurn,
       end,
       behavior,
-      performClip,
+      performAction,
       onLifecycleData,
       registerDataPublisher,
       registerTurnSender,
@@ -2931,4 +3164,4 @@ function createProxyClient(options) {
   };
 }
 
-export { AdaptivePlayoutController, AvatarCall, AvatarVideoSurface, DEFAULT_APPROACHING_END_LEAD_SECONDS, DEFAULT_AVATAR_PLAYOUT_DELAY_SECONDS, DEFAULT_CREDITS_LOW_LEAD_SECONDS, DEFAULT_GOVERNOR_CONFIG, DEFAULT_GRACE_CEILING_SECONDS, DEFAULT_GRACE_WINDOW_LEAD_SECONDS, DEFAULT_IDLE_SECONDS, DEFAULT_IDLE_WARN_LEAD_SECONDS, DEFAULT_TURN_TIMEOUT_SECONDS, MAX_RECONNECT_ATTEMPTS, MAX_SESSION_INSTRUCTIONS_CHARS, MIC_LEASE_ENDED_TIMEOUT_MS, RECONNECT_BACKOFF_MS, RealtimeAvatarCapacityError, RealtimeAvatarLiveKitRoom, SessionLifecycleRoomBridge, capacityErrorFromBusy, capacityStateFromGrant, createProxyClient, knownBehaviorStates, mapTurnState, readInboundRtp, sessionBehaviorSchema, sessionClipSchema, shouldReplayPendingTurn, splitCallTranscript, useAvatarAdaptivePlayoutDelay, useAvatarCall, useAvatarPlayoutDelay, useAvatarQualityGovernor, useCallTranscript, useLiveKitAvatarGrant, useMicLease, useRealtimeSession, useReleaseMicLeaseOnTrackEnded, useSessionLifecycle };
+export { AdaptivePlayoutController, AvatarCall, AvatarVideoSurface, DEFAULT_APPROACHING_END_LEAD_SECONDS, DEFAULT_AVATAR_PLAYOUT_DELAY_SECONDS, DEFAULT_CREDITS_LOW_LEAD_SECONDS, DEFAULT_GOVERNOR_CONFIG, DEFAULT_GRACE_CEILING_SECONDS, DEFAULT_GRACE_WINDOW_LEAD_SECONDS, DEFAULT_IDLE_SECONDS, DEFAULT_IDLE_WARN_LEAD_SECONDS, DEFAULT_TURN_TIMEOUT_SECONDS, MAX_RECONNECT_ATTEMPTS, MAX_SESSION_INSTRUCTIONS_CHARS, MIC_LEASE_ENDED_TIMEOUT_MS, RECONNECT_BACKOFF_MS, RealtimeAvatarCapacityError, RealtimeAvatarLiveKitRoom, SessionLifecycleRoomBridge, capacityErrorFromBusy, capacityStateFromGrant, createProxyClient, knownBehaviorStates, mapTurnState, readInboundRtp, sessionBehaviorSchema, sessionClipSchema, shouldReplayPendingTurn, splitCallTranscript, useAvatarAdaptivePlayoutDelay, useAvatarCall, useAvatarPlayoutDelay, useAvatarQualityGovernor, useCallTranscript, useCharacterTools, useLiveKitAvatarGrant, useMicLease, useRealtimeSession, useReleaseMicLeaseOnTrackEnded, useSessionLifecycle };

package/dist/server.d.ts

@@ -1,2 +1,3 @@
-export { RealtimeAvatar, RealtimeAvatarError, RealtimeAvatarHttpError, RealtimeAvatarOptions, StartCallOptions, verifyTranscript } from './index.js';
-export { i as Asset, h as AssetKind, A as Avatar, m as AvatarClip, c as AvatarSourceSwap, b as AvatarUpdate, n as CallConnection, C as CallMode, a as CallPolicy, o as CallQueued, d as ClipDeclaration, f as ClipLibrary, p as ClipLibraryPlan, e as ClipLibraryUpdate, q as ClipSource, g as ClipSyncResult, r as ContextMessage, l as CreditBalance, E as EndCallOptions, s as EndCallReason, j as ListSessionsOptions, L as LoopRedirect, S as StartCallResult, T as TranscriptPayload, k as UsageSession, U as UsageSessionPage, V as VideoPolicy, t as VideoState, u as isQueued } from './types-D_aIKv4P.js';
+export { RealtimeAvatar, RealtimeAvatarError, RealtimeAvatarHttpError, RealtimeAvatarOptions, StartCallOptions, clipLibraryDeclarationSchema, verifyTranscript } from './index.js';
+export { h as Asset, g as AssetKind, A as Avatar, l as AvatarClip, c as AvatarSourceSwap, b as AvatarUpdate, m as CallConnection, C as CallMode, a as CallPolicy, n as CallQueued, o as ClipDeclaration, f as ClipLibrary, d as ClipLibraryDeclaration, p as ClipLibraryPlan, e as ClipLibraryUpdate, q as ClipSource, r as ContextMessage, k as CreditBalance, E as EndCallOptions, s as EndCallReason, i as ListSessionsOptions, L as LoopRedirect, S as StartCallResult, T as TranscriptPayload, j as UsageSession, U as UsageSessionPage, V as VideoPolicy, t as isQueued } from './types-FoJMJimV.js';
+import 'zod';