realtime-avatar 0.9.0 → 0.10.0

package/dist/react-native.js

@@ -2398,11 +2398,11 @@ function useRealtimeSession(input) {
     requestGracefulClose();
     lifecycle.reset();
   }, [lifecycle, requestGracefulClose]);
-  const performClip = useCallback(
-    (clipId, opts) => {
+  const performAction = useCallback(
+    (actionId, opts) => {
       const requestId = newTurnId();
       const publish = dataPublisherRef.current;
-      const trimmed = clipId.trim();
+      const trimmed = actionId.trim();
       if (!publish || !trimmed) {
         const result = { requestId, accepted: false, reason: "not_connected" };
         cbRef.current.onClipResult?.(result);
@@ -2417,10 +2417,9 @@ function useRealtimeSession(input) {
         }, opts?.timeoutMs ?? 5e3);
         pendingClipsRef.current.set(requestId, { resolve, timer });
         publish({
-          kind: "clip_request",
+          kind: "action_request",
           request_id: requestId,
-          clip_id: trimmed,
-          ...typeof opts?.holdSeconds === "number" ? { hold_seconds: opts.holdSeconds } : {}
+          action_id: trimmed
         });
       });
     },
@@ -2465,7 +2464,7 @@ function useRealtimeSession(input) {
       retryTurn,
       end,
       behavior,
-      performClip,
+      performAction,
       onLifecycleData,
       registerDataPublisher,
       registerTurnSender,
@@ -2487,7 +2486,7 @@ function useRealtimeSession(input) {
       retryTurn,
       end,
       behavior,
-      performClip,
+      performAction,
       onLifecycleData,
       registerDataPublisher,
       registerTurnSender,

package/dist/react.d.ts

@@ -1,10 +1,25 @@
+import { RegisteredTool, ToolRegistration } from './tools.js';
 import { ReactNode } from 'react';
-import { g as AvatarSessionClient, A as AvatarVideoFit } from './proxy-client-DDqYRz-_.js';
-export { a as AdaptivePlayoutController, b as AdaptivePlayoutDecision, c as AdaptivePlayoutOptions, d as AdaptivePlayoutSample, e as ApproachingEndEvent, f as ApproachingEndReason, aK as AvatarVideoSurface, aL as AvatarVideoSurfaceProps, B as BehaviorSnapshot, C as CallTranscript, h as CallTranscriptSegment, i as CartesiaTtsModel, j as ClipResult, k as ClosingTurnResult, l as CreditsLowEvent, D as DEFAULT_APPROACHING_END_LEAD_SECONDS, m as DEFAULT_CREDITS_LOW_LEAD_SECONDS, n as DEFAULT_GOVERNOR_CONFIG, o as DEFAULT_GRACE_CEILING_SECONDS, p as DEFAULT_GRACE_WINDOW_LEAD_SECONDS, q as DEFAULT_IDLE_SECONDS, r as DEFAULT_IDLE_WARN_LEAD_SECONDS, s as DEFAULT_TURN_TIMEOUT_SECONDS, E as EndReason, t as EndedEvent, u as ExtendResult, F as FishTtsModel, v as FreezeReadingFn, G as Governor, w as GovernorAction, x as GovernorConfig, y as GovernorSignal, z as GovernorState, H as GraceWindowClosedEvent, I as GraceWindowOpenEvent, J as GraceWindowState, K as IdleWarningEvent, M as InboundRtpCursor, N as InboundRtpReading, O as KnownBehaviorState, P as LLMProvider, Q as LLMSelection, R as LiveKitAvatarGrantState, S as LiveKitAvatarGrantStatus, T as LiveKitCapacityState, U as LiveKitConnectionStatus, V as LiveKitSessionRequest, W as LiveKitSessionStartResult, aM as LivePlaybackKeeper, X as MAX_RECONNECT_ATTEMPTS, Y as MAX_SESSION_INSTRUCTIONS_CHARS, aN as MIC_LEASE_ENDED_TIMEOUT_MS, aO as MicLease, aP as PlayableVideoElement, Z as ProxyClientOptions, _ as QualityCap, $ as RECONNECT_BACKOFF_MS, aQ as RealtimeAvatarCapacityError, aR as RealtimeAvatarLiveKitRoom, aS as RealtimeAvatarLiveKitRoomProps, a0 as RealtimeAvatarRequestOptions, a1 as RealtimeSessionApi, a2 as RealtimeSessionMedia, a3 as RealtimeSessionRoomSinks, a4 as ReconnectPolicy, a5 as ReconnectingEvent, a6 as RecoveryState, a7 as RetryStep, a8 as SessionBehavior, a9 as SessionClip, aa as SessionClocks, ab as SessionEndReason, ac as SessionLifecycleApi, ad as SessionLifecyclePhase, ae as SessionLifecyclePhaseKind, af as SessionLifecycleRoomBridge, ag as SessionLifecycleRoomBridgeProps, ah as SurfaceLayers, ai as TurnState, aj as TurnTimeoutEvent, ak as UseAvatarQualityGovernorInput, al as UseLiveKitAvatarGrantInput, am as UseRealtimeSessionInput, an as UseSessionLifecycleInput, ao as VoiceSpec, ap as VoiceSpecInput, aq as capacityErrorFromBusy, ar as capacityStateFromGrant, as as createProxyClient, au as knownBehaviorStates, av as mapTurnState, aw as readInboundRtp, ax as sessionBehaviorSchema, ay as sessionClipSchema, aT as shouldReplayPendingTurn, az as splitCallTranscript, aA as useAvatarAdaptivePlayoutDelay, aB as useAvatarPlayoutDelay, aC as useAvatarQualityGovernor, aD as useCallTranscript, aE as useLiveKitAvatarGrant, aG as useMicLease, aH as useRealtimeSession, aI as useReleaseMicLeaseOnTrackEnded, aJ as useSessionLifecycle } from './proxy-client-DDqYRz-_.js';
+import { g as AvatarSessionClient, A as AvatarVideoFit } from './proxy-client-B6OnXpaC.js';
+export { a as AdaptivePlayoutController, b as AdaptivePlayoutDecision, c as AdaptivePlayoutOptions, d as AdaptivePlayoutSample, e as ApproachingEndEvent, f as ApproachingEndReason, aK as AvatarVideoSurface, aL as AvatarVideoSurfaceProps, B as BehaviorSnapshot, C as CallTranscript, h as CallTranscriptSegment, i as CartesiaTtsModel, j as ClipResult, k as ClosingTurnResult, l as CreditsLowEvent, D as DEFAULT_APPROACHING_END_LEAD_SECONDS, m as DEFAULT_CREDITS_LOW_LEAD_SECONDS, n as DEFAULT_GOVERNOR_CONFIG, o as DEFAULT_GRACE_CEILING_SECONDS, p as DEFAULT_GRACE_WINDOW_LEAD_SECONDS, q as DEFAULT_IDLE_SECONDS, r as DEFAULT_IDLE_WARN_LEAD_SECONDS, s as DEFAULT_TURN_TIMEOUT_SECONDS, E as EndReason, t as EndedEvent, u as ExtendResult, F as FishTtsModel, v as FreezeReadingFn, G as Governor, w as GovernorAction, x as GovernorConfig, y as GovernorSignal, z as GovernorState, H as GraceWindowClosedEvent, I as GraceWindowOpenEvent, J as GraceWindowState, K as IdleWarningEvent, M as InboundRtpCursor, N as InboundRtpReading, O as KnownBehaviorState, P as LLMProvider, Q as LLMSelection, R as LiveKitAvatarGrantState, S as LiveKitAvatarGrantStatus, T as LiveKitCapacityState, U as LiveKitConnectionStatus, V as LiveKitSessionRequest, W as LiveKitSessionStartResult, aM as LivePlaybackKeeper, X as MAX_RECONNECT_ATTEMPTS, Y as MAX_SESSION_INSTRUCTIONS_CHARS, aN as MIC_LEASE_ENDED_TIMEOUT_MS, aO as MicLease, aP as PlayableVideoElement, Z as ProxyClientOptions, _ as QualityCap, $ as RECONNECT_BACKOFF_MS, aQ as RealtimeAvatarCapacityError, aR as RealtimeAvatarLiveKitRoom, aS as RealtimeAvatarLiveKitRoomProps, a0 as RealtimeAvatarRequestOptions, a1 as RealtimeSessionApi, a2 as RealtimeSessionMedia, a3 as RealtimeSessionRoomSinks, a4 as ReconnectPolicy, a5 as ReconnectingEvent, a6 as RecoveryState, a7 as RetryStep, a8 as SessionBehavior, a9 as SessionClip, aa as SessionClocks, ab as SessionEndReason, ac as SessionLifecycleApi, ad as SessionLifecyclePhase, ae as SessionLifecyclePhaseKind, af as SessionLifecycleRoomBridge, ag as SessionLifecycleRoomBridgeProps, ah as SurfaceLayers, ai as TurnState, aj as TurnTimeoutEvent, ak as UseAvatarQualityGovernorInput, al as UseLiveKitAvatarGrantInput, am as UseRealtimeSessionInput, an as UseSessionLifecycleInput, ao as VoiceSpec, ap as VoiceSpecInput, aq as capacityErrorFromBusy, ar as capacityStateFromGrant, as as createProxyClient, au as knownBehaviorStates, av as mapTurnState, aw as readInboundRtp, ax as sessionBehaviorSchema, ay as sessionClipSchema, aT as shouldReplayPendingTurn, az as splitCallTranscript, aA as useAvatarAdaptivePlayoutDelay, aB as useAvatarPlayoutDelay, aC as useAvatarQualityGovernor, aD as useCallTranscript, aE as useLiveKitAvatarGrant, aG as useMicLease, aH as useRealtimeSession, aI as useReleaseMicLeaseOnTrackEnded, aJ as useSessionLifecycle } from './proxy-client-B6OnXpaC.js';
 export { D as DEFAULT_AVATAR_PLAYOUT_DELAY_SECONDS } from './playout-delay-DWOc8iCl.js';
+import 'zod';
 import '@livekit/components-react';
 import 'livekit-client';
-import 'zod';
+
+declare function useCharacterTools(tools: Record<string, RegisteredTool>, options?: {
+    onResult?: (event: {
+        name: string;
+        callId: string;
+        ok: boolean;
+        error?: string;
+    }) => void;
+}): {
+    status: "idle" | "registering" | "ready" | "error";
+    registered: string[];
+    rejected: ToolRegistration["rejected"];
+    error?: string;
+};
 
 /**
  * The CONNECT surface (client half) — one component that puts a live character on screen.
@@ -105,4 +120,4 @@ declare function useAvatarCall(props: AvatarCallProps): {
 /** One component: a live character on screen. */
 declare function AvatarCall(props: AvatarCallProps): ReactNode;
 
-export { AvatarCall, type AvatarCallEndReason, type AvatarCallHandle, type AvatarCallProps, type AvatarCallStatus, AvatarSessionClient, AvatarVideoFit, useAvatarCall };
+export { AvatarCall, type AvatarCallEndReason, type AvatarCallHandle, type AvatarCallProps, type AvatarCallStatus, AvatarSessionClient, AvatarVideoFit, useAvatarCall, useCharacterTools };

package/dist/react.js

@@ -1,10 +1,204 @@
 "use client";
 import { useRef, useState, useEffect, useCallback, useMemo, createElement, Fragment } from 'react';
-import { useTranscriptions, useLocalParticipant, RoomAudioRenderer, LiveKitRoom, useMaybeRoomContext, useVoiceAssistant, useConnectionState, VideoTrack, useRoomContext, useChat } from '@livekit/components-react';
-import { ConnectionQuality, RoomEvent, ConnectionState, TrackEvent, Track, VideoQuality, ConnectionError, ConnectionErrorReason, DisconnectReason, Room } from 'livekit-client';
+import { useRoomContext, useConnectionState, useTranscriptions, useLocalParticipant, RoomAudioRenderer, LiveKitRoom, useMaybeRoomContext, useVoiceAssistant, VideoTrack, useChat } from '@livekit/components-react';
+import { ConnectionState, ConnectionQuality, RoomEvent, TrackEvent, Track, VideoQuality, ConnectionError, ConnectionErrorReason, DisconnectReason, Room } from 'livekit-client';
 import { z } from 'zod';
 
-// ../client/src/react/avatar-call.ts
+// ../client/src/react/use-character-tools.ts
+var invocationSchema = z.object({
+  name: z.string().regex(/^[a-zA-Z0-9_-]{1,64}$/),
+  args: z.string().max(8192),
+  call_id: z.string().min(1).max(128)
+}).strict();
+var registrationSchema = z.object({
+  accepted: z.array(z.string()),
+  rejected: z.array(z.object({ name: z.string(), reason: z.string() })).default([])
+});
+var MAX_TOOLS = 32;
+var MAX_MANIFEST_BYTES = 12e3;
+var MAX_RESULT_CHARS = 8192;
+var TOOL_DEADLINE_MS = 2500;
+var REGISTER_METHOD = "rta.tools.register";
+var INVOKE_METHOD = "rta.tool.invoke";
+var NAME_RE = /^[a-zA-Z0-9_-]{1,64}$/;
+function candidateIdentities(room) {
+  const all = Array.from(room.remoteParticipants.values(), (p) => p.identity).filter(Boolean);
+  const agents = all.filter((id) => /^agent[-_]/i.test(id));
+  return [...agents, ...all.filter((id) => !agents.includes(id))];
+}
+function methodMissing(error) {
+  const code = error?.code;
+  if (code === "UNSUPPORTED_METHOD" || code === "RECIPIENT_NOT_FOUND" || code === 1401 || code === 1402) {
+    return true;
+  }
+  const message = error instanceof Error ? error.message : String(error ?? "");
+  return /not supported|unsupported_method|recipient_not_found|recipient not found/i.test(message);
+}
+var sleep = (ms) => new Promise((resolve) => setTimeout(resolve, ms));
+function buildManifest(tools) {
+  const descriptors = [];
+  const rejected = [];
+  for (const [name, tool] of Object.entries(tools)) {
+    if (!NAME_RE.test(name)) {
+      rejected.push({ name, reason: "name must match [a-zA-Z0-9_-]{1,64}" });
+      continue;
+    }
+    if (!tool?.description) {
+      rejected.push({ name, reason: "description is required \u2014 it is how she knows when to call it" });
+      continue;
+    }
+    if (typeof tool.execute !== "function") {
+      rejected.push({ name, reason: "execute must be a function" });
+      continue;
+    }
+    if (descriptors.length >= MAX_TOOLS) {
+      rejected.push({ name, reason: `over the ${MAX_TOOLS}-tool limit for one session` });
+      continue;
+    }
+    descriptors.push({ name, description: tool.description, parameters: tool.parameters ?? {} });
+  }
+  return { descriptors, rejected };
+}
+async function attachAvatarTools(room, tools, options = {}) {
+  const { descriptors, rejected } = buildManifest(tools);
+  const payload = JSON.stringify({ tools: descriptors });
+  if (payload.length > MAX_MANIFEST_BYTES) {
+    throw new Error(
+      `tool manifest is ${payload.length} bytes, over the ${MAX_MANIFEST_BYTES} limit \u2014 shorten descriptions or register fewer tools`
+    );
+  }
+  options.signal?.throwIfAborted();
+  let disposed = false;
+  let activeCaller;
+  const inFlight = /* @__PURE__ */ new Set();
+  const dispose = () => {
+    if (disposed) return;
+    disposed = true;
+    room.localParticipant.unregisterRpcMethod?.(INVOKE_METHOD);
+    for (const call of inFlight) call.abort();
+    options.signal?.removeEventListener("abort", dispose);
+  };
+  options.signal?.addEventListener("abort", dispose, { once: true });
+  room.localParticipant.registerRpcMethod(INVOKE_METHOD, async ({ payload: payload2, callerIdentity }) => {
+    let name = "", callId = "";
+    try {
+      if (disposed) throw new Error("Tool registration was disposed");
+      if (callerIdentity && callerIdentity !== activeCaller) throw new Error("Unregistered tool caller");
+      const req = invocationSchema.parse(JSON.parse(payload2));
+      name = req.name;
+      callId = req.call_id;
+      const tool = tools[name];
+      if (!tool) return JSON.stringify({ ok: false, call_id: callId, error: `no tool named ${name}` });
+      const args = z.record(z.string(), z.unknown()).parse(req.args ? JSON.parse(req.args) : {});
+      options.onInvoke?.(name, args);
+      const run = tool.execute;
+      const controller = new AbortController();
+      inFlight.add(controller);
+      const timer = setTimeout(() => controller.abort(), TOOL_DEADLINE_MS);
+      let result;
+      try {
+        result = await run(args, { signal: controller.signal, callId });
+        controller.signal.throwIfAborted();
+      } finally {
+        inFlight.delete(controller);
+        clearTimeout(timer);
+      }
+      const encoded = typeof result === "string" ? result : JSON.stringify(result ?? null);
+      if (encoded.length > MAX_RESULT_CHARS) {
+        return JSON.stringify({
+          ok: false,
+          call_id: callId,
+          error: `result was ${encoded.length} chars, over the ${MAX_RESULT_CHARS} limit`
+        });
+      }
+      options.onResult?.({ name, callId, ok: true });
+      return JSON.stringify({ ok: true, call_id: callId, result: encoded });
+    } catch (error) {
+      options.onResult?.({ name, callId, ok: false, error: error instanceof Error ? error.message : "tool failed" });
+      return JSON.stringify({
+        ok: false,
+        call_id: callId,
+        error: error instanceof Error ? error.message : "tool failed"
+      });
+    }
+  });
+  const deadline = Date.now() + Math.max(0, options.timeoutMs ?? 8e3);
+  let lastError = null;
+  let sawCandidate = false;
+  try {
+    for (; ; ) {
+      options.signal?.throwIfAborted();
+      for (const identity of candidateIdentities(room)) {
+        sawCandidate = true;
+        activeCaller = identity;
+        try {
+          const raw = await room.localParticipant.performRpc({
+            destinationIdentity: identity,
+            method: REGISTER_METHOD,
+            payload
+          });
+          options.signal?.throwIfAborted();
+          const result = registrationSchema.parse(JSON.parse(raw));
+          return {
+            accepted: result.accepted,
+            rejected: [...rejected, ...result.rejected],
+            dispose
+          };
+        } catch (error) {
+          activeCaller = void 0;
+          lastError = error;
+          if (!methodMissing(error)) throw error;
+        }
+      }
+      if (Date.now() >= deadline) break;
+      await sleep(200);
+    }
+    throw new Error(
+      sawCandidate ? `no participant accepted the tool manifest \u2014 either this session was not minted with the \`client_tools\` capability (a server-side grant), or the agent never armed registration within ${options.timeoutMs ?? 8e3}ms` + (lastError instanceof Error ? ` (last error: ${lastError.message})` : "") : "no remote participant joined the room before the timeout \u2014 nothing to register with"
+    );
+  } catch (error) {
+    dispose();
+    throw error;
+  }
+}
+
+// ../client/src/react/use-character-tools.ts
+function useCharacterTools(tools, options = {}) {
+  const room = useRoomContext();
+  const connection = useConnectionState();
+  const callback = useRef(options.onResult);
+  callback.current = options.onResult;
+  const [state, setState] = useState({ status: "idle", registered: [], rejected: [] });
+  useEffect(() => {
+    if (connection !== ConnectionState.Connected) {
+      setState({ status: "idle", registered: [], rejected: [] });
+      return;
+    }
+    const abort = new AbortController();
+    setState({ status: "registering", registered: [], rejected: [] });
+    void attachAvatarTools(room, tools, {
+      signal: abort.signal,
+      onResult: (event) => {
+        if (!abort.signal.aborted) callback.current?.(event);
+      }
+    }).then((result) => {
+      if (abort.signal.aborted) {
+        result.dispose();
+        return;
+      }
+      setState({ status: "ready", registered: result.accepted, rejected: result.rejected });
+    }).catch((error) => {
+      if (!abort.signal.aborted) setState({
+        status: "error",
+        registered: [],
+        rejected: [],
+        error: error instanceof Error ? error.message : "Tool registration failed"
+      });
+    });
+    return () => abort.abort();
+  }, [room, tools, connection]);
+  return state;
+}
 var holder = null;
 var queue = [];
 var pendingEnded = null;
@@ -978,6 +1172,33 @@ function useAvatarQualityGovernor(input) {
   }, [enabled, room, videoTrack, freezeReading, config, tickMs]);
 }
 
+// ../client/src/react/frame-recovery.ts
+var FrameRecovery = class {
+  constructor(stallAfterMs) {
+    this.stallAfterMs = stallAfterMs;
+  }
+  stallAfterMs;
+  lastFrameAtMs = null;
+  stableSinceMs = null;
+  ready = true;
+  stall() {
+    this.ready = false;
+    this.stableSinceMs = null;
+  }
+  frame(nowMs) {
+    const gapMs = this.lastFrameAtMs === null ? 0 : nowMs - this.lastFrameAtMs;
+    if (this.lastFrameAtMs !== null && gapMs > this.stallAfterMs) this.stall();
+    if (!this.ready) {
+      if (this.stableSinceMs === null || gapMs > Math.min(350, this.stallAfterMs)) {
+        this.stableSinceMs = nowMs;
+      }
+      this.ready = nowMs - this.stableSinceMs >= 500;
+    }
+    this.lastFrameAtMs = nowMs;
+    return this.ready;
+  }
+};
+
 // ../client/src/react/avatar-video-surface.ts
 var DEFAULT_AVATAR_FRAME_STALL_MS = 800;
 var AVATAR_FRAME_GAP_FREEZE_FLOOR_MS = 100;
@@ -1000,6 +1221,8 @@ function AvatarVideoSurface(props) {
     onPlayoutDelayChange
   } = props;
   const testId = props["data-testid"];
+  const visibilityCallback = useRef(props.onLiveVisibilityChange);
+  visibilityCallback.current = props.onLiveVisibilityChange;
   const { videoTrack, audioTrack } = useVoiceAssistant();
   useAvatarPlayoutDelay(videoTrack, audioTrack);
   const playoutDelaySeconds = useAvatarAdaptivePlayoutDelay(
@@ -1042,10 +1265,18 @@ function AvatarVideoSurface(props) {
     liveWanted,
     connected && !networkStalled ? idleReturnDelayMs : 0
   );
+  useEffect(() => {
+    visibilityCallback.current?.(showLive);
+  }, [showLive]);
   const liveDims = videoTrack?.publication?.dimensions ?? null;
   const idleVideoRef = useRef(null);
   const boxAspect = aspectRatio ?? null;
-  useIdleWhileResting(idleVideoRef, idleVideoUrl, showLive);
+  useIdleWhileResting(
+    idleVideoRef,
+    idleVideoUrl,
+    showLive,
+    !live || connectionState === ConnectionState.Disconnected
+  );
   useLiveResumeOnProducing(liveWrapRef, trackProducing);
   const fitClass = fit === "cover" ? AVATAR_VIDEO_FIT_COVER : AVATAR_VIDEO_FIT_CONTAIN;
   const liveLabel = showLive && liveDims && liveDims.width > 0 && liveDims.height > 0 ? `${liveDims.width}\xD7${liveDims.height}` : null;
@@ -1177,6 +1408,7 @@ function isFrameFreezeInhibited(snapshot) {
 }
 function useLiveFrameFlow(wrapRef, trackProducing, trackIdentity, stallAfterMs) {
   const boundedStallMs = normalizeFrameStallMs(stallAfterMs);
+  const recovery = useMemo(() => new FrameRecovery(boundedStallMs), [trackIdentity, boundedStallMs]);
   const [flowing, setFlowing] = useState(false);
   const [seenFrame, setSeenFrame] = useState(false);
   const flowingRef = useRef(false);
@@ -1206,6 +1438,7 @@ function useLiveFrameFlow(wrapRef, trackProducing, trackIdentity, stallAfterMs)
     const markFrame = () => {
       const previous = sampleRef.current;
       const now = Date.now();
+      lastCurrentTime = video?.currentTime ?? lastCurrentTime;
       const firstFrame = !previous.seenFrame;
       sampleRef.current = {
         seenFrame: true,
@@ -1219,9 +1452,10 @@ function useLiveFrameFlow(wrapRef, trackProducing, trackIdentity, stallAfterMs)
         resumePending: false
       };
       if (firstFrame) setSeenFrame(true);
-      if (!flowingRef.current) {
-        flowingRef.current = true;
-        setFlowing(true);
+      const nextFlowing = recovery.frame(now);
+      if (flowingRef.current !== nextFlowing) {
+        flowingRef.current = nextFlowing;
+        setFlowing(nextFlowing);
       }
     };
     const cancelFrameCallback = () => {
@@ -1263,12 +1497,14 @@ function useLiveFrameFlow(wrapRef, trackProducing, trackIdentity, stallAfterMs)
         lastCurrentTime = video.currentTime;
         markFrame();
       }
-      const nextFlowing = isFrameFlowingAt({
+      const frameFresh = isFrameFlowingAt({
         seenFrame: sampleRef.current.seenFrame,
         lastFrameAtMs: sampleRef.current.lastFrameAtMs,
         nowMs: Date.now(),
         stallAfterMs: boundedStallMs
       });
+      if (!frameFresh && sampleRef.current.seenFrame) recovery.stall();
+      const nextFlowing = frameFresh && recovery.ready;
       if (flowingRef.current !== nextFlowing) {
         flowingRef.current = nextFlowing;
         setFlowing(nextFlowing);
@@ -1281,7 +1517,7 @@ function useLiveFrameFlow(wrapRef, trackProducing, trackIdentity, stallAfterMs)
       stopObserving();
       cancelFrameCallback();
     };
-  }, [wrapRef, trackProducing, trackIdentity, boundedStallMs]);
+  }, [wrapRef, trackProducing, trackIdentity, boundedStallMs, recovery]);
   const freezeReading = useCallback(() => {
     const sample = sampleRef.current;
     const lastFrameAtMs = sample.lastFrameAtMs;
@@ -1320,19 +1556,19 @@ function useDebouncedHide(wanted, delayMs) {
   }, [wanted, delayMs]);
   return shown;
 }
-function useIdleWhileResting(videoRef, idleVideoUrl, showLive) {
+function useIdleWhileResting(videoRef, idleVideoUrl, showLive, resetToAnchor) {
   useEffect(() => {
     const video = videoRef.current;
     if (!video || !idleVideoUrl) return;
-    applyIdleRest(video, showLive);
-  }, [videoRef, idleVideoUrl, showLive]);
+    applyIdleRest(video, showLive, resetToAnchor);
+  }, [videoRef, idleVideoUrl, showLive, resetToAnchor]);
 }
-function applyIdleRest(video, showLive) {
+function applyIdleRest(video, showLive, resetToAnchor = true) {
   if (showLive) {
     video.pause();
     return;
   }
-  if (video.currentTime !== 0) video.currentTime = 0;
+  if (resetToAnchor && video.currentTime !== 0) video.currentTime = 0;
   if (video.paused) void Promise.resolve(video.play()).catch(() => {
   });
 }
@@ -2641,11 +2877,11 @@ function useRealtimeSession(input) {
     requestGracefulClose();
     lifecycle.reset();
   }, [lifecycle, requestGracefulClose]);
-  const performClip = useCallback(
-    (clipId, opts) => {
+  const performAction = useCallback(
+    (actionId, opts) => {
       const requestId = newTurnId();
       const publish = dataPublisherRef.current;
-      const trimmed = clipId.trim();
+      const trimmed = actionId.trim();
       if (!publish || !trimmed) {
         const result = { requestId, accepted: false, reason: "not_connected" };
         cbRef.current.onClipResult?.(result);
@@ -2660,10 +2896,9 @@ function useRealtimeSession(input) {
         }, opts?.timeoutMs ?? 5e3);
         pendingClipsRef.current.set(requestId, { resolve, timer });
         publish({
-          kind: "clip_request",
+          kind: "action_request",
           request_id: requestId,
-          clip_id: trimmed,
-          ...typeof opts?.holdSeconds === "number" ? { hold_seconds: opts.holdSeconds } : {}
+          action_id: trimmed
         });
       });
     },
@@ -2708,7 +2943,7 @@ function useRealtimeSession(input) {
       retryTurn,
       end,
       behavior,
-      performClip,
+      performAction,
       onLifecycleData,
       registerDataPublisher,
       registerTurnSender,
@@ -2730,7 +2965,7 @@ function useRealtimeSession(input) {
       retryTurn,
       end,
       behavior,
-      performClip,
+      performAction,
       onLifecycleData,
       registerDataPublisher,
       registerTurnSender,
@@ -2894,4 +3129,4 @@ function createProxyClient(options) {
   };
 }
 
-export { AdaptivePlayoutController, AvatarCall, AvatarVideoSurface, DEFAULT_APPROACHING_END_LEAD_SECONDS, DEFAULT_AVATAR_PLAYOUT_DELAY_SECONDS, DEFAULT_CREDITS_LOW_LEAD_SECONDS, DEFAULT_GOVERNOR_CONFIG, DEFAULT_GRACE_CEILING_SECONDS, DEFAULT_GRACE_WINDOW_LEAD_SECONDS, DEFAULT_IDLE_SECONDS, DEFAULT_IDLE_WARN_LEAD_SECONDS, DEFAULT_TURN_TIMEOUT_SECONDS, MAX_RECONNECT_ATTEMPTS, MAX_SESSION_INSTRUCTIONS_CHARS, MIC_LEASE_ENDED_TIMEOUT_MS, RECONNECT_BACKOFF_MS, RealtimeAvatarCapacityError, RealtimeAvatarLiveKitRoom, SessionLifecycleRoomBridge, capacityErrorFromBusy, capacityStateFromGrant, createProxyClient, knownBehaviorStates, mapTurnState, readInboundRtp, sessionBehaviorSchema, sessionClipSchema, shouldReplayPendingTurn, splitCallTranscript, useAvatarAdaptivePlayoutDelay, useAvatarCall, useAvatarPlayoutDelay, useAvatarQualityGovernor, useCallTranscript, useLiveKitAvatarGrant, useMicLease, useRealtimeSession, useReleaseMicLeaseOnTrackEnded, useSessionLifecycle };
+export { AdaptivePlayoutController, AvatarCall, AvatarVideoSurface, DEFAULT_APPROACHING_END_LEAD_SECONDS, DEFAULT_AVATAR_PLAYOUT_DELAY_SECONDS, DEFAULT_CREDITS_LOW_LEAD_SECONDS, DEFAULT_GOVERNOR_CONFIG, DEFAULT_GRACE_CEILING_SECONDS, DEFAULT_GRACE_WINDOW_LEAD_SECONDS, DEFAULT_IDLE_SECONDS, DEFAULT_IDLE_WARN_LEAD_SECONDS, DEFAULT_TURN_TIMEOUT_SECONDS, MAX_RECONNECT_ATTEMPTS, MAX_SESSION_INSTRUCTIONS_CHARS, MIC_LEASE_ENDED_TIMEOUT_MS, RECONNECT_BACKOFF_MS, RealtimeAvatarCapacityError, RealtimeAvatarLiveKitRoom, SessionLifecycleRoomBridge, capacityErrorFromBusy, capacityStateFromGrant, createProxyClient, knownBehaviorStates, mapTurnState, readInboundRtp, sessionBehaviorSchema, sessionClipSchema, shouldReplayPendingTurn, splitCallTranscript, useAvatarAdaptivePlayoutDelay, useAvatarCall, useAvatarPlayoutDelay, useAvatarQualityGovernor, useCallTranscript, useCharacterTools, useLiveKitAvatarGrant, useMicLease, useRealtimeSession, useReleaseMicLeaseOnTrackEnded, useSessionLifecycle };

package/dist/server.d.ts

@@ -1,2 +1,3 @@
-export { RealtimeAvatar, RealtimeAvatarError, RealtimeAvatarHttpError, RealtimeAvatarOptions, StartCallOptions, verifyTranscript } from './index.js';
-export { i as Asset, h as AssetKind, A as Avatar, m as AvatarClip, c as AvatarSourceSwap, b as AvatarUpdate, n as CallConnection, C as CallMode, a as CallPolicy, o as CallQueued, d as ClipDeclaration, f as ClipLibrary, p as ClipLibraryPlan, e as ClipLibraryUpdate, q as ClipSource, g as ClipSyncResult, r as ContextMessage, l as CreditBalance, E as EndCallOptions, s as EndCallReason, j as ListSessionsOptions, L as LoopRedirect, S as StartCallResult, T as TranscriptPayload, k as UsageSession, U as UsageSessionPage, V as VideoPolicy, t as VideoState, u as isQueued } from './types-D_aIKv4P.js';
+export { RealtimeAvatar, RealtimeAvatarError, RealtimeAvatarHttpError, RealtimeAvatarOptions, StartCallOptions, clipLibraryDeclarationSchema, verifyTranscript } from './index.js';
+export { h as Asset, g as AssetKind, A as Avatar, l as AvatarClip, c as AvatarSourceSwap, b as AvatarUpdate, m as CallConnection, C as CallMode, a as CallPolicy, n as CallQueued, o as ClipDeclaration, f as ClipLibrary, d as ClipLibraryDeclaration, p as ClipLibraryPlan, e as ClipLibraryUpdate, q as ClipSource, r as ContextMessage, k as CreditBalance, E as EndCallOptions, s as EndCallReason, i as ListSessionsOptions, L as LoopRedirect, S as StartCallResult, T as TranscriptPayload, j as UsageSession, U as UsageSessionPage, V as VideoPolicy, t as isQueued } from './types-BgYFwlSn.js';
+import 'zod';

package/dist/server.js

@@ -1 +1 @@
-export { RealtimeAvatar, RealtimeAvatarError, RealtimeAvatarHttpError, isQueued, verifyTranscript } from './chunk-IQUJLPYF.js';
+export { RealtimeAvatar, RealtimeAvatarError, RealtimeAvatarHttpError, clipLibraryDeclarationSchema, isQueued, verifyTranscript } from './chunk-5UEMZN6J.js';

package/dist/tanstack-start.d.ts

@@ -1,5 +1,5 @@
-import { P as ProxyConfig } from './types-QoMHpR1h.js';
-import './types-D_aIKv4P.js';
+import { P as ProxyConfig } from './types-zQtYeR_l.js';
+import './types-BgYFwlSn.js';
 
 /**
  * TanStack Start. Mount at `routes/api/realtime-avatar/$.ts` — the trailing `$` is Start's

package/dist/tanstack-start.js

@@ -1,5 +1,5 @@
-import { createProxyHandler } from './chunk-A2ATKWJT.js';
-import './chunk-IQUJLPYF.js';
+import { createProxyHandler } from './chunk-UJQBYKPI.js';
+import './chunk-5UEMZN6J.js';
 
 // src/tanstack-start.ts
 function realtimeAvatarServerRoute(config) {

package/dist/tools.d.ts

@@ -1,3 +1,5 @@
+import { z } from 'zod';
+
 /**
  * The browser tool plane, without React.
  *
@@ -13,6 +15,17 @@
  *    minted with the `client_tools` capability, which your server decides. If registration
  *    reports the method is missing, the mint did not grant it — the fix is server-side.
  */
+
+declare function defineAvatarTool<I extends z.ZodType, O extends z.ZodType>(definition: {
+    description: string;
+    inputSchema: I;
+    outputSchema: O;
+    execute: (input: z.output<I>, context: ToolContext) => z.input<O> | Promise<z.input<O>>;
+}): {
+    description: string;
+    parameters: z.core.ZodStandardJSONSchemaPayload<I>;
+    execute(input: unknown, context: ToolContext): Promise<z.output<O>>;
+};
 /** Caps the wire enforces. Exceeding them is a rejection, not a truncation. */
 declare const MAX_TOOLS = 32;
 declare const MAX_MANIFEST_BYTES = 12000;
@@ -60,13 +73,16 @@ interface ToolRegistration {
         name: string;
         reason: string;
     }>;
+    dispose: () => void;
 }
 /** The subset of a LiveKit Room this needs — kept structural so it is trivial to fake. */
 interface RoomLike {
     localParticipant: {
         registerRpcMethod(method: string, handler: (data: {
             payload: string;
+            callerIdentity?: string;
         }) => Promise<string>): void;
+        unregisterRpcMethod?(method: string): void;
         performRpc(options: {
             destinationIdentity: string;
             method: string;
@@ -104,8 +120,15 @@ interface RoomLike {
  */
 declare function attachAvatarTools(room: RoomLike, tools: Record<string, RegisteredTool>, options?: {
     onInvoke?: (name: string, args: unknown) => void;
+    onResult?: (event: {
+        name: string;
+        callId: string;
+        ok: boolean;
+        error?: string;
+    }) => void;
+    signal?: AbortSignal;
     /** How long to keep waiting for the agent to arm its side. Default 8000ms. */
     timeoutMs?: number;
 }): Promise<ToolRegistration>;
 
-export { type AvatarTool, MAX_MANIFEST_BYTES, MAX_RESULT_CHARS, MAX_TOOLS, type RegisteredTool, TOOL_DEADLINE_MS, type ToolContext, type ToolRegistration, attachAvatarTools };
+export { type AvatarTool, MAX_MANIFEST_BYTES, MAX_RESULT_CHARS, MAX_TOOLS, type RegisteredTool, TOOL_DEADLINE_MS, type ToolContext, type ToolRegistration, attachAvatarTools, defineAvatarTool };