realtime-avatar 0.3.0

package/README.md

@@ -0,0 +1,176 @@
+# realtime-avatar
+
+A live character your users can talk to — voice, or voice and video. She listens while she
+speaks, so you can interrupt her mid-sentence and she stops, the way a person stops.
+
+```bash
+npm install realtime-avatar
+```
+
+```ts
+import { RealtimeAvatar, isQueued } from "realtime-avatar";
+
+const rta = new RealtimeAvatar({ apiKey: process.env.REALTIME_AVATAR_API_KEY! });
+
+// On your server. The client picks WHO to call; you decide everything about the call.
+const call = await rta.startCall({
+  avatarId: "ava_…",
+  instructions: "You are Rin. Short, warm, specific sentences.",
+  maxSeconds: 600,
+});
+
+if (isQueued(call)) return { queued: true, position: call.position };
+return call.raw;   // relay to the browser byte-for-byte
+```
+
+That is the whole server half. The client joins with the payload and renders her.
+
+---
+
+## What you get
+
+| | |
+| --- | --- |
+| **Voice or video** | The same call, the same code. `mode: "voice"` is audio-only and cheaper. |
+| **Full-duplex** | Interrupt her and she stops. A cough does not derail her. A pause is not the end of your turn. |
+| **Your character** | Your footage, your voice, your persona — not a stock presenter. |
+| **Your tools** | You run them; you feed the result back into a turn. Nothing calls your API on your behalf. |
+| **Priced to leave on** | Under $5/hour of live conversation, billed by the second. |
+
+---
+
+## The shape of an integration
+
+```
+your client  ──▶  your backend  ──▶  Realtime Avatar
+                  (holds the key,      (capacity, listening,
+                   decides the call)    thinking, speaking, rendering)
+     ◀────────── live audio + video, and she is listening ──────────
+```
+
+Two facts follow from that picture and drive everything else:
+
+1. **The key is server-only.** A browser holding it can start unlimited calls on your
+   account. The constructor throws in a browser runtime so that fails loudly, not silently.
+2. **The connection payload is opaque.** Relay `call.raw` untouched — the browser client
+   validates it strictly.
+
+---
+
+## API
+
+Everything is on one class. The full types are in
+[`libs/http-client/src/types.ts`](../http-client/src/types.ts) — one file, no import chasing.
+Almost every shape in it is **derived** from the published OpenAPI contract rather than declared
+beside it, so a field that changes upstream fails the typecheck here. The exceptions are named at
+the top of that file with the reason for each: two are gaps in the contract itself (it declares no
+query parameters for `GET /v1/usage/sessions`, and the transcript webhook body is not in it at
+all), and the `video` policy types are deliberately not one-to-one with the wire.
+
+```ts
+// calls
+rta.startCall({ avatarId, mode?, instructions?, context?, maxSeconds?, video?, transcript?, metadata? })
+rta.endCall(sessionId, { reason? })    // free an abandoned call's slot now; idempotent, never throws
+
+// avatars
+rta.createAvatarFromVideo({ displayName, videoUrl, voice? })
+rta.listAvatars()
+rta.getAvatar(avatarId)
+rta.syncClips(avatarId, clipUrls)      // after ANY clip change
+
+// assets
+rta.createRemoteAsset({ kind, remoteUrl })
+rta.uploadAsset(file, { kind })
+
+// billing
+rta.creditBalance()
+
+// webhooks
+verifyTranscript(rawBytes, headers, secret)
+```
+
+---
+
+## Subpaths
+
+**One install.** tsup treeshakes per entry, so a server-only app importing `realtime-avatar/server`
+gets 18.8 KB with no React and no LiveKit in it, even though they sit in the same tarball.
+
+Server — these hold your API key:
+
+| Import | What it is |
+| --- | --- |
+| `realtime-avatar` | `RealtimeAvatar`, `isQueued`, `verifyTranscript`, the two error classes |
+| `realtime-avatar/server` | The same client with no route adapters — 18.8 KB |
+| `realtime-avatar/nextjs` | `createRealtimeAvatarRoute` — App Router `{ GET, POST }` |
+| `realtime-avatar/hono` | `realtimeAvatarHono` — Hono, Workers, Bun, Deno |
+| `realtime-avatar/express` | `realtimeAvatarExpress` |
+| `realtime-avatar/tanstack-start` | TanStack Start server route |
+
+Browser — these never can:
+
+| Import | What it is |
+| --- | --- |
+| `realtime-avatar/react` | `AvatarCall`, `useAvatarCall`, `useRealtimeSession`, `useSessionLifecycle` |
+| `realtime-avatar/react-native` | The same surface for Expo / React Native |
+| `realtime-avatar/browser` | `enableMicrophone`, `attachRemoteAudio` — no React |
+| `realtime-avatar/tools` | `attachAvatarTools` — the browser tool plane |
+
+Every adapter takes the same two hooks: `authorize` gates the request, `session` decides the
+call. Policy — `instructions`, `maxSeconds`, `voice`, `video` — is decided in `session`, on
+your server. A route that spreads the request body into `startCall` hands your caller your
+system prompt and your bill.
+
+### Importing a server entry into a browser build throws
+
+Not a lint rule and not a naming convention — the six server subpaths carry `browser` and
+`react-native` export conditions pointing at a module whose only statement is a `throw`, so the
+key-holding code never enters a client module graph. Measured: a browser bundle that imports both
+halves contains **0** occurrences of `Bearer` or `apiKey`.
+
+This lived under a second npm name (`realtime-avatar-react`) until 2026-08-26, on the theory that a
+condition "chooses which file is bundled, never whether the package is". That was tested and is
+false. Two things worth knowing if you copy the pattern: do **not** use `"browser": null` — Vite 8
+and rolldown ignore it and bundle the server file *with* the secret — and do not leave
+`"sideEffects": false` in place, which lets a bundler treeshake a throw-only module away and
+silently disarms the whole guard.
+
+## The two subpaths that are not React
+
+`enableMicrophone` returns the cause as a value instead of throwing, because "the mic won't
+start" is one sentence covering six causes with different fixes — and one of them, a macOS
+system denial, cannot be fixed from the address bar and needs the browser restarted.
+`attachRemoteAudio` attaches into the DOM *before* `connect`, which is what stops a track
+arriving mid-connect from being lost on a fast connection.
+
+`attachAvatarTools` runs your functions in the page. Nothing is executed on the platform, and
+a tool has **2.5 seconds** to answer before the call gives up on it and tells her it failed.
+
+## What `/react` exports, and what it deliberately does not
+
+31 names, down from 82 on 2026-08-26. Two groups came out and are not coming back:
+
+**LiveKit symbols.** `Room`, `RoomEvent`, `Track`, `useRoomContext` and 20 others were
+re-exported from here. `livekit-client` and `@livekit/components-react` are peer dependencies, so
+import them from LiveKit directly and you get the version you installed — re-exporting put their
+types in this package's public surface, which meant a LiveKit major could break ours without a
+line of our code changing.
+
+**State-machine internals.** `acquireMicLease`, `stepQualityGovernor`, `retryStep`,
+`resolveWarnBeforeMs` and 27 more were the individual steps the hooks drive. None was callable in
+a useful order from outside, and every one was a name we would have had to keep working forever.
+
+What stayed: the components and hooks, the `DEFAULT_*` constants (so you can read the timings
+rather than guess them), the two capacity mappers `capacityErrorFromBusy` / `capacityStateFromGrant`
+for building your own queue UI, and the zod schemas `sessionBehaviorSchema` / `sessionClipSchema`.
+
+
+---
+
+## Docs and support
+
+- Full documentation: <https://realtimeavatar.ai/docs>
+- Issues and feature requests: this repo
+- The API is versioned at `/api/v1`; breaking changes get a new version, not a silent edit.
+
+MIT licensed.

package/dist/browser.d.ts

@@ -0,0 +1,203 @@
+/**
+ * Turning the microphone on, and saying what went wrong when it does not.
+ *
+ * WHY THIS EXISTS. `room.localParticipant.setMicrophoneEnabled(true)` is one line, so every
+ * integration writes it inline and none of them handle its rejection — the reject path is
+ * invisible until a real user is on a real machine. When it fails there is nothing to see:
+ * the call is already connected, the page is mid-`await`, and the rejection becomes an
+ * unhandled promise nobody is looking at. The reported symptom is always the same sentence —
+ * "the mic won't start" — and it covers at least six unrelated causes with different fixes,
+ * one of which is not in the browser at all.
+ *
+ * So this returns a RESULT rather than throwing: the cause becomes a value the caller has to
+ * handle, and each one carries the sentence that names its actual fix. `message` is what the
+ * browser said; `hint` is what the person should do.
+ *
+ * It is deliberately structural — it never imports `livekit-client`, so it pins no version
+ * and adds no bytes. Anything with a `localParticipant.setMicrophoneEnabled` satisfies it.
+ */
+/** The sliver of livekit-client's `Room` this needs. Structural on purpose — see above. */
+interface MicrophoneCapableRoom {
+    localParticipant: {
+        setMicrophoneEnabled(enabled: boolean): Promise<unknown>;
+    };
+}
+/**
+ * Why the microphone did not start. Every variant has a DIFFERENT fix, which is the whole
+ * reason this is an enum and not a boolean.
+ */
+type MicrophoneFailureReason = 
+/** No `navigator.mediaDevices` — the page is not a secure context. */
+"insecure-origin"
+/** The permission prompt was never answered. getUserMedia may never settle; we time out. */
+ | "no-answer"
+/** The BROWSER denied it — the site's own permission, resettable from the address bar. */
+ | "denied-by-browser"
+/** The OS denied it — macOS System Settings, nothing the page or the site can change. */
+ | "denied-by-os"
+/** No input device is connected, or none matches the constraints. */
+ | "no-device"
+/** A device exists but something else holds it — another tab, app, or a stale call. */
+ | "device-in-use"
+/** Anything unrecognised. `message` still carries what the browser said. */
+ | "unknown";
+type MicrophoneResult = {
+    ok: true;
+} | {
+    ok: false;
+    reason: MicrophoneFailureReason;
+    /** What the browser reported, verbatim. Show it — it is often exact. */
+    message: string;
+    /** What the person should actually do about it. */
+    hint: string;
+};
+interface EnableMicrophoneOptions {
+    /**
+     * How long to wait before calling it a no-answer. Default 15s.
+     *
+     * This is not belt-and-braces. Per the getUserMedia specification a prompt the user never
+     * dismisses may leave the promise permanently pending — "neither resolve nor reject" — and
+     * a page that awaits it in its connect path hangs on a state that has no error and no end.
+     * A deadline turns that into a reportable outcome.
+     */
+    timeoutMs?: number;
+}
+/**
+ * Publish the caller's microphone, and report WHY if it does not start.
+ *
+ * ```ts
+ * const mic = await enableMicrophone(room);
+ * if (!mic.ok) {
+ *   status.textContent = `Microphone: ${mic.message}`;
+ *   help.textContent = mic.hint;          // the sentence that names the fix
+ * }
+ * ```
+ *
+ * Never throws. A call that cannot start the microphone is an ordinary outcome of asking a
+ * person for a device, not a programmer error, so it belongs in the return type.
+ *
+ * Note it does NOT hang up on failure — the call is still live and still billing, and only
+ * the caller knows whether a text-only session is useful to them. If it is not, disconnect
+ * in the `!ok` branch; leaving a room connected against a device the user never granted is
+ * how the retry ends up contending with the call that is still holding it.
+ */
+declare function enableMicrophone(room: MicrophoneCapableRoom, options?: EnableMicrophoneOptions): Promise<MicrophoneResult>;
+
+/**
+ * Making the character AUDIBLE — the half of "audio works" that has nothing to do with the
+ * microphone.
+ *
+ * WHY THIS EXISTS. Two failures here look identical to a developer ("she never speaks") and
+ * neither raises anything:
+ *
+ *  1. The audio element is created but never inserted into the document. `track.attach()`
+ *     hands back a detached `<audio>`; a detached element is not reliably played by every
+ *     engine, and there is no node on the page to fall back to. Nothing errors — the call is
+ *     connected, the track is subscribed, the meter moves, and it is silent.
+ *  2. Autoplay is blocked. A page that has not yet had a user gesture may not start audio,
+ *     so the first call a visitor makes is mute. `room.startAudio()` fixes it, but it must be
+ *     called FROM a gesture — which means the page needs a button, which means the page has
+ *     to know it is blocked. That signal is an event nobody subscribes to.
+ *
+ * So this owns both: it attaches into the DOM, and it tells you when a gesture is required
+ * and hands you the closure that spends it.
+ *
+ * Structural like its sibling — no `livekit-client` import, no version pin.
+ */
+/** The sliver of a livekit-client `Track` this needs. */
+interface AttachableTrack {
+    kind: string;
+    attach(): HTMLMediaElement;
+    detach(): HTMLMediaElement[];
+}
+/** The sliver of a livekit-client `Room` this needs. */
+interface AudioCapableRoom {
+    canPlaybackAudio: boolean;
+    startAudio(): Promise<void>;
+    on(event: string, listener: (...args: never[]) => void): unknown;
+    off(event: string, listener: (...args: never[]) => void): unknown;
+}
+interface AttachRemoteAudioOptions {
+    /**
+     * Where to park the audio elements. Defaults to `document.body`.
+     *
+     * They are not `display:none` — a hidden media element is still subject to the same
+     * autoplay rules, and hiding it only removes the browser's own affordance.
+     */
+    container?: HTMLElement;
+    /**
+     * Called when the browser refuses to start audio without a gesture, and again with `null`
+     * once audio is playing. Render a button from it and call `unblock` in the click handler:
+     *
+     * ```ts
+     * onPlaybackBlocked: (unblock) => {
+     *   button.hidden = unblock === null;
+     *   button.onclick = () => unblock?.();
+     * }
+     * ```
+     */
+    onPlaybackBlocked?: (unblock: (() => Promise<void>) | null) => void;
+}
+interface RemoteAudioAttachment {
+    /** Stop listening and remove every element this created. Safe to call twice. */
+    detach(): void;
+}
+/**
+ * Play every remote audio track the room subscribes to, and surface an autoplay block.
+ *
+ * Call it BEFORE `room.connect()` — a track subscribed during connect is missed otherwise,
+ * and that race is the version of this bug that only reproduces on a fast connection.
+ *
+ * ```ts
+ * const audio = attachRemoteAudio(room, {
+ *   onPlaybackBlocked: (unblock) => {
+ *     enableSound.hidden = unblock === null;
+ *     enableSound.onclick = () => unblock?.();
+ *   },
+ * });
+ * await room.connect(url, token);
+ * // …later
+ * audio.detach();
+ * ```
+ */
+declare function attachRemoteAudio(room: AudioCapableRoom, options?: AttachRemoteAudioOptions): RemoteAudioAttachment;
+
+/**
+ * Ping-pong (boomerang) playback for idle/ambient clips.
+ *
+ * Idle videos are plain image-to-video generations (start frame != end
+ * frame), so a hard `loop` visibly jumps at the wrap. Playing forward, then
+ * scrubbing back to the start, makes ANY clip loop seamlessly — no seamless
+ * A==B generation required. Reverse is done by stepping `currentTime` on a
+ * requestAnimationFrame clock because `playbackRate < 0` is unsupported in
+ * most browsers.
+ */
+type BoomerangPlaybackOptions = {
+    /** Reverse-phase speed relative to real time. Default 1 (same pace as forward). */
+    reverseRate?: number;
+    /** Seconds from each end at which the direction flips. Default 0.05. */
+    edgeEpsilonSeconds?: number;
+};
+type BoomerangPlayback = {
+    /** Stop driving the element (leaves the video itself untouched). */
+    stop(): void;
+    /** Current loop state, e.g. to hand playback position to a live renderer. */
+    position(): {
+        timeSeconds: number;
+        direction: "forward" | "reverse";
+    };
+    /** Jump the loop to a position+direction (seamless handback from a live render). */
+    alignTo(timeSeconds: number, direction: "forward" | "reverse"): void;
+};
+/**
+ * Drive a `<video>` element in a forward-then-reverse loop. The element should
+ * be muted + playsInline (autoplay policies). Idempotent per call; returns a
+ * handle whose `stop()` cancels the drive loop. Safe with elements whose
+ * metadata has not loaded yet.
+ *
+ *     const playback = attachBoomerangPlayback(videoEl);
+ *     // later: playback.stop();
+ */
+declare function attachBoomerangPlayback(video: HTMLVideoElement, options?: BoomerangPlaybackOptions): BoomerangPlayback;
+
+export { type AttachRemoteAudioOptions, type AttachableTrack, type AudioCapableRoom, type BoomerangPlayback, type BoomerangPlaybackOptions, type EnableMicrophoneOptions, type MicrophoneCapableRoom, type MicrophoneFailureReason, type MicrophoneResult, type RemoteAudioAttachment, attachBoomerangPlayback, attachRemoteAudio, enableMicrophone };

package/dist/browser.js

@@ -0,0 +1,172 @@
+// ../browser/src/microphone.ts
+var HINTS = {
+  "insecure-origin": "The browser only exposes microphones on a secure origin. Serve the page over https, or open it on http://localhost \u2014 a LAN address like http://192.168.1.5 will not do.",
+  "no-answer": "The permission prompt is probably still open, or was dismissed without an answer. Look for it in the address bar, answer it, and try again.",
+  "denied-by-browser": "This site is blocked from using the microphone. Click the icon at the left of the address bar, set Microphone to Allow, and reload.",
+  "denied-by-os": "The operating system is blocking the browser itself, so no site can record. On macOS: System Settings > Privacy & Security > Microphone, enable your browser, then RESTART it \u2014 the change does not apply to a running browser.",
+  "no-device": "No microphone is available. Connect one, and if you are on Bluetooth headphones check they are in a mode that has a microphone.",
+  "device-in-use": "Another application or browser tab is holding the microphone. Close it \u2014 including any call from this page you did not hang up \u2014 and try again.",
+  unknown: "Try again, and if it persists include the message above in your report."
+};
+function isOperatingSystemDenial(message) {
+  return /\bby system\b|system permission/i.test(message);
+}
+function classify(error) {
+  if (error instanceof TypeError && /mediaDevices|undefined/i.test(error.message)) {
+    return { reason: "insecure-origin", message: error.message };
+  }
+  const name = typeof error === "object" && error !== null && "name" in error ? String(error.name) : "";
+  const message = typeof error === "object" && error !== null && "message" in error && error.message ? String(error.message) : String(error);
+  switch (name) {
+    case "NotAllowedError":
+    case "PermissionDeniedError":
+      return {
+        reason: isOperatingSystemDenial(message) ? "denied-by-os" : "denied-by-browser",
+        message
+      };
+    case "NotFoundError":
+    case "DevicesNotFoundError":
+    case "OverconstrainedError":
+    case "ConstraintNotSatisfiedError":
+      return { reason: "no-device", message };
+    case "NotReadableError":
+    case "TrackStartError":
+    case "AbortError":
+      return { reason: "device-in-use", message };
+    case "SecurityError":
+      return { reason: "insecure-origin", message };
+    default:
+      return { reason: "unknown", message };
+  }
+}
+function fail(reason, message) {
+  return { ok: false, reason, message, hint: HINTS[reason] };
+}
+async function enableMicrophone(room, options = {}) {
+  const timeoutMs = options.timeoutMs ?? 15e3;
+  if (typeof navigator === "undefined" || !navigator.mediaDevices) {
+    return fail(
+      "insecure-origin",
+      "navigator.mediaDevices is undefined \u2014 this page is not a secure context."
+    );
+  }
+  const NO_ANSWER = /* @__PURE__ */ Symbol("no-answer");
+  let timer;
+  try {
+    const deadline = new Promise((resolve) => {
+      timer = setTimeout(() => resolve(NO_ANSWER), timeoutMs);
+    });
+    const outcome = await Promise.race([
+      room.localParticipant.setMicrophoneEnabled(true).then(() => "granted"),
+      deadline
+    ]);
+    if (outcome === NO_ANSWER) {
+      return fail(
+        "no-answer",
+        `The microphone permission prompt was not answered within ${Math.round(timeoutMs / 1e3)}s.`
+      );
+    }
+    return { ok: true };
+  } catch (error) {
+    const { reason, message } = classify(error);
+    return fail(reason, message);
+  } finally {
+    if (timer !== void 0) clearTimeout(timer);
+  }
+}
+
+// ../browser/src/remote-audio.ts
+var AUDIO_PLAYBACK_CHANGED = "audioPlaybackChanged";
+var TRACK_SUBSCRIBED = "trackSubscribed";
+var AUDIO_KIND = "audio";
+function attachRemoteAudio(room, options = {}) {
+  const container = options.container ?? document.body;
+  const elements = /* @__PURE__ */ new Set();
+  let detached = false;
+  const publishPlaybackState = () => {
+    if (detached) return;
+    options.onPlaybackBlocked?.(room.canPlaybackAudio ? null : () => room.startAudio());
+  };
+  const onTrackSubscribed = (track) => {
+    if (detached || track.kind !== AUDIO_KIND) return;
+    const element = track.attach();
+    element.autoplay = true;
+    container.appendChild(element);
+    elements.add(element);
+    publishPlaybackState();
+  };
+  room.on(TRACK_SUBSCRIBED, onTrackSubscribed);
+  room.on(AUDIO_PLAYBACK_CHANGED, publishPlaybackState);
+  publishPlaybackState();
+  return {
+    detach() {
+      if (detached) return;
+      detached = true;
+      room.off(TRACK_SUBSCRIBED, onTrackSubscribed);
+      room.off(AUDIO_PLAYBACK_CHANGED, publishPlaybackState);
+      for (const element of elements) element.remove();
+      elements.clear();
+    }
+  };
+}
+
+// ../client/src/browser/boomerang.ts
+function attachBoomerangPlayback(video, options = {}) {
+  const reverseRate = options.reverseRate ?? 1;
+  const edge = options.edgeEpsilonSeconds ?? 0.05;
+  let raf = null;
+  let direction = 1;
+  let last = now();
+  const step = (nowMs) => {
+    raf = requestAnimationFrame(step);
+    const duration = video.duration;
+    if (!Number.isFinite(duration) || duration <= 0) {
+      last = nowMs;
+      return;
+    }
+    if (direction === 1) {
+      if (video.paused) void video.play().catch(() => {
+      });
+      if (video.currentTime >= duration - edge) {
+        direction = -1;
+        video.pause();
+        last = nowMs;
+      }
+      return;
+    }
+    const dt = Math.min(0.1, (nowMs - last) / 1e3) * reverseRate;
+    last = nowMs;
+    const next = video.currentTime - dt;
+    if (next <= edge) {
+      video.currentTime = 0;
+      direction = 1;
+    } else {
+      video.currentTime = next;
+    }
+  };
+  raf = requestAnimationFrame(step);
+  return {
+    stop() {
+      if (raf !== null) {
+        cancelAnimationFrame(raf);
+        raf = null;
+      }
+    },
+    position() {
+      return { timeSeconds: video.currentTime, direction: direction === 1 ? "forward" : "reverse" };
+    },
+    alignTo(timeSeconds, nextDirection) {
+      const duration = video.duration;
+      const clamped = Number.isFinite(duration) && duration > 0 ? Math.min(Math.max(timeSeconds, 0), duration - edge) : Math.max(timeSeconds, 0);
+      video.currentTime = clamped;
+      direction = nextDirection === "reverse" ? -1 : 1;
+      last = now();
+      if (direction === -1) video.pause();
+    }
+  };
+}
+function now() {
+  return typeof performance !== "undefined" ? performance.now() : Date.now();
+}
+
+export { attachBoomerangPlayback, attachRemoteAudio, enableMicrophone };

package/dist/express.d.ts

@@ -0,0 +1,28 @@
+import { P as ProxyConfig } from './types-E8SrD6sv.js';
+import './types-C_EMPwN7.js';
+
+type Expressish = {
+    method: string;
+    originalUrl: string;
+    headers: Record<string, string | string[] | undefined>;
+    body?: unknown;
+};
+type ResponseLike = {
+    status(code: number): ResponseLike;
+    set(field: string, value: string): ResponseLike;
+    send(body: string): void;
+};
+/**
+ * Express 4/5.
+ *
+ * ```ts
+ * app.use("/api/realtime-avatar", express.json(), realtimeAvatarExpress({ apiKey, session }));
+ * ```
+ *
+ * Express hands us a parsed body, so it is re-serialized here rather than streamed. That is
+ * fine for this route — the payloads are small — but it is why the Fetch adapters are the
+ * better path if you have a choice.
+ */
+declare function realtimeAvatarExpress(config: ProxyConfig): (req: Expressish, res: ResponseLike) => Promise<void>;
+
+export { realtimeAvatarExpress };