realtime-avatar 0.15.0 → 0.17.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +82 -2
- package/dist/{chunk-LZMJJX7C.js → chunk-27BB4DQ2.js} +1 -1
- package/dist/chunk-MCIXU6XP.js +33 -0
- package/dist/{chunk-PKKTFUA7.js → chunk-OESXLLQB.js} +407 -13
- package/dist/connection-history-CXL5QE56.js +1 -0
- package/dist/connection-history.d.ts +121 -0
- package/dist/connection-history.js +1 -0
- package/dist/express.d.ts +3 -2
- package/dist/express.js +3 -2
- package/dist/hono.d.ts +3 -2
- package/dist/hono.js +3 -2
- package/dist/index.d.ts +12 -3
- package/dist/index.js +2 -1
- package/dist/nextjs.d.ts +3 -2
- package/dist/nextjs.js +3 -2
- package/dist/{proxy-client-BGPikYWc.d.ts → proxy-client-DmJVj18Z.d.ts} +59 -4
- package/dist/react-native.d.ts +2 -2
- package/dist/react-native.js +191 -6
- package/dist/react.d.ts +2 -2
- package/dist/react.js +191 -6
- package/dist/recording.d.ts +135 -0
- package/dist/recording.js +43 -0
- package/dist/server.d.ts +2 -1
- package/dist/server.js +2 -1
- package/dist/tanstack-start.d.ts +3 -2
- package/dist/tanstack-start.js +3 -2
- package/dist/{types-B9GTrpx0.d.ts → types-BnlnrYgN.d.ts} +303 -5
- package/dist/{types-CJcTowDB.d.ts → types-CFkiNV3f.d.ts} +1 -1
- package/package.json +9 -1
package/README.md
CHANGED
|
@@ -4,7 +4,7 @@ A live character your users can talk to — voice, or voice and video. She liste
|
|
|
4
4
|
speaks, so you can interrupt her mid-sentence and she stops, the way a person stops.
|
|
5
5
|
|
|
6
6
|
```bash
|
|
7
|
-
npm install --save-exact realtime-avatar@0.
|
|
7
|
+
npm install --save-exact realtime-avatar@0.17.0
|
|
8
8
|
```
|
|
9
9
|
|
|
10
10
|
```ts
|
|
@@ -25,6 +25,79 @@ return call.raw; // relay to the browser byte-for-byte
|
|
|
25
25
|
|
|
26
26
|
That is the whole server half. The client joins with the payload and renders her.
|
|
27
27
|
|
|
28
|
+
## Optional call recordings
|
|
29
|
+
|
|
30
|
+
Your server decides whether to record after your application obtains consent:
|
|
31
|
+
|
|
32
|
+
```ts
|
|
33
|
+
const call = await rta.startCall({ avatarId, recording: "audio_video" });
|
|
34
|
+
if (!isQueued(call) && call.recording) {
|
|
35
|
+
await saveRecordingId(call.sessionId, call.recording.recordingId);
|
|
36
|
+
}
|
|
37
|
+
|
|
38
|
+
// Later, in your authenticated admin backend:
|
|
39
|
+
const recording = await rta.getRecording(recordingId);
|
|
40
|
+
if (recording.status === "ready") {
|
|
41
|
+
const { url, expiresAt } = await rta.getRecordingAccess(recordingId);
|
|
42
|
+
// Return this short-lived access to the authorized viewer.
|
|
43
|
+
}
|
|
44
|
+
```
|
|
45
|
+
|
|
46
|
+
Omitted or `"off"` disables recording. `"audio"` records published user and avatar audio;
|
|
47
|
+
`"video"` records published video without audio; `"audio_video"` keeps both on one media timeline.
|
|
48
|
+
Recording does not enable the microphone, camera, or screen sharing. Only tracks that participants
|
|
49
|
+
authorize and publish can be recorded. Camera controls remain a future SDK feature.
|
|
50
|
+
|
|
51
|
+
Recordings may finish processing after a call ends. Use `listRecordings({ sessionId })` to find
|
|
52
|
+
them, or refresh `getRecording(recordingId)` while processing. These methods and
|
|
53
|
+
`getRecordingAccess` require a server key with `recordings:read`.
|
|
54
|
+
|
|
55
|
+
Save `recordingId`, not a playback URL. Files are retained until `retainedUntil` (30 days by
|
|
56
|
+
default); each URL expires at `expiresAt` (up to one hour, capped by retention). Obtain fresh
|
|
57
|
+
access before replaying or seeking after expiry. An expired URL does not delete the file.
|
|
58
|
+
Treat the URL as private: anyone who has it can play that file until it expires.
|
|
59
|
+
|
|
60
|
+
## Optional connection history
|
|
61
|
+
|
|
62
|
+
Set `connectionHistory: true` in the server's `startCall` policy when the caller needs a
|
|
63
|
+
tenant-scoped LiveKit quality timeline. The SDK enables its collector only when RTA grants the
|
|
64
|
+
single-session capability. It receives the bridge's existing LiveKit connection snapshots,
|
|
65
|
+
deduplicates changes, batches at most 32 observations, and stops at 240; it does not poll WebRTC
|
|
66
|
+
statistics or own reconnect logic. Upload failures are bounded and never interrupt a call.
|
|
67
|
+
|
|
68
|
+
```ts
|
|
69
|
+
const call = await rta.startCall({ avatarId, connectionHistory: true });
|
|
70
|
+
// Relay call.raw unchanged. The browser SDK uploads capability-gated observations automatically.
|
|
71
|
+
const history = await rta.getConnectionHistory(call.sessionId);
|
|
72
|
+
```
|
|
73
|
+
|
|
74
|
+
History is an observation of native connection state and publisher quality. It is not a recording,
|
|
75
|
+
packet-loss report, or proof that a device rendered or played a track. The read requires
|
|
76
|
+
`usage:read`; store the session ID and fetch history from your authorized backend.
|
|
77
|
+
|
|
78
|
+
Transcript delivery remains the signed `transcript` webhook configured on `startCall`. Join the
|
|
79
|
+
transcript and recordings by `sessionId`; keep your own script revision with that call. Transcript
|
|
80
|
+
timestamps describe conversation turns and do not by themselves establish frame-accurate media
|
|
81
|
+
alignment for lip-sync analysis.
|
|
82
|
+
|
|
83
|
+
Client-safe Zod schemas and derived types are available from `realtime-avatar/recording`.
|
|
84
|
+
They are the same executable contract used by the service, verified against the published digest.
|
|
85
|
+
|
|
86
|
+
```mermaid
|
|
87
|
+
---
|
|
88
|
+
title: Recording ownership and private playback
|
|
89
|
+
---
|
|
90
|
+
flowchart LR
|
|
91
|
+
App[Application server: consent and recording policy] --> RTA[RTA: call and recording lifecycle]
|
|
92
|
+
RTA --> Media[Media provider: record published tracks]
|
|
93
|
+
Media --> Storage[Private media storage]
|
|
94
|
+
RTA --> Metadata[Recording ID and session ID]
|
|
95
|
+
Admin[Authorized admin backend] --> RTA
|
|
96
|
+
RTA --> Access[Temporary playback URL]
|
|
97
|
+
Access --> Player[Video or audio player]
|
|
98
|
+
Storage --> Player
|
|
99
|
+
```
|
|
100
|
+
|
|
28
101
|
New here? The [quickstart](https://realtimeavatar.ai/docs/quickstart) goes from an API key to a
|
|
29
102
|
working call, and a [sandbox key](https://realtimeavatar.ai/signup) is free with no card. Mount
|
|
30
103
|
the server half on your framework:
|
|
@@ -82,9 +155,15 @@ all), and the `video` policy types are deliberately not one-to-one with the wire
|
|
|
82
155
|
|
|
83
156
|
```ts
|
|
84
157
|
// calls
|
|
85
|
-
rta.startCall({ avatarId, mode?, instructions?, context?, maxSeconds?, video?, transcript?, metadata? })
|
|
158
|
+
rta.startCall({ avatarId, mode?, instructions?, context?, maxSeconds?, video?, recording?, connectionHistory?, transcript?, metadata? })
|
|
86
159
|
rta.endCall(sessionId, { reason? }) // free an abandoned call's slot; idempotent, never throws
|
|
87
160
|
|
|
161
|
+
// optional recordings; server only, requires recordings:read
|
|
162
|
+
rta.listRecordings({ sessionId, limit?, cursor? })
|
|
163
|
+
rta.getRecording(recordingId)
|
|
164
|
+
rta.getRecordingAccess(recordingId)
|
|
165
|
+
rta.getConnectionHistory(sessionId) // bounded LiveKit observations; requires usage:read
|
|
166
|
+
|
|
88
167
|
// avatars
|
|
89
168
|
rta.createAvatarFromImage({ displayName, imageUrl, motionPrompt?, voice? }) // the only lane
|
|
90
169
|
rta.createAvatarFromVideo({ displayName, videoUrl, voice? }) // DEPRECATED — closed, 422
|
|
@@ -141,6 +220,7 @@ Browser — these never can:
|
|
|
141
220
|
| `realtime-avatar/react-native` | The same surface for Expo / React Native |
|
|
142
221
|
| `realtime-avatar/browser` | `enableMicrophone`, `attachRemoteAudio`, `prepareAvatarRoom` — no React |
|
|
143
222
|
| `realtime-avatar/tools` | `attachAvatarTools` — the browser tool plane |
|
|
223
|
+
| `realtime-avatar/recording` | Client-safe Zod recording schemas and derived types |
|
|
144
224
|
|
|
145
225
|
Every adapter takes the same two hooks: `authorize` gates the request, `session` decides the
|
|
146
226
|
call. Policy — `instructions`, `maxSeconds`, `voice`, `video` — is decided in `session`, on
|
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
import { RealtimeAvatar, isQueued, RealtimeAvatarHttpError } from './chunk-
|
|
1
|
+
import { RealtimeAvatar, isQueued, RealtimeAvatarHttpError } from './chunk-OESXLLQB.js';
|
|
2
2
|
|
|
3
3
|
// ../proxy/src/config.ts
|
|
4
4
|
var json = (body, status = 200) => new Response(JSON.stringify(body), {
|
|
@@ -0,0 +1,33 @@
|
|
|
1
|
+
import { z } from 'zod';
|
|
2
|
+
|
|
3
|
+
// ../http-client/src/generated/connection-history.ts
|
|
4
|
+
var ConnectionQuality = { Excellent: "excellent", Good: "good", Poor: "poor", Lost: "lost", Unknown: "unknown" };
|
|
5
|
+
var ConnectionState = { Disconnected: "disconnected", Connecting: "connecting", Connected: "connected", Reconnecting: "reconnecting", SignalReconnecting: "signalReconnecting" };
|
|
6
|
+
var MAX_CONNECTION_HISTORY_OBSERVATIONS = 240;
|
|
7
|
+
var MAX_CONNECTION_HISTORY_BATCH = 32;
|
|
8
|
+
var CONNECTION_HISTORY_RETENTION_MS = 30 * 24 * 60 * 60 * 1e3;
|
|
9
|
+
var quality = z.enum(ConnectionQuality);
|
|
10
|
+
var connectionHistoryObservationSchema = z.object({
|
|
11
|
+
sequence: z.number().int().min(1).max(MAX_CONNECTION_HISTORY_OBSERVATIONS),
|
|
12
|
+
elapsedMs: z.number().int().min(0).max(864e5),
|
|
13
|
+
clientObservedAt: z.string().datetime({ offset: true }).nullable(),
|
|
14
|
+
connectionState: z.enum(ConnectionState),
|
|
15
|
+
localQuality: quality,
|
|
16
|
+
audioQuality: quality.nullable(),
|
|
17
|
+
videoQuality: quality.nullable()
|
|
18
|
+
}).strict();
|
|
19
|
+
var connectionHistoryGrantSchema = z.object({
|
|
20
|
+
endpoint: z.string().url().startsWith("https://"),
|
|
21
|
+
token: z.string().min(32).max(4096),
|
|
22
|
+
expiresAt: z.string().datetime({ offset: true })
|
|
23
|
+
}).strict();
|
|
24
|
+
var connectionHistoryUploadSchema = z.object({
|
|
25
|
+
sessionId: z.string().min(1).max(200),
|
|
26
|
+
observations: z.array(connectionHistoryObservationSchema).min(1).max(MAX_CONNECTION_HISTORY_BATCH)
|
|
27
|
+
}).strict();
|
|
28
|
+
var connectionHistoryResponseSchema = z.object({
|
|
29
|
+
sessionId: z.string().min(1).max(200),
|
|
30
|
+
observations: z.array(connectionHistoryObservationSchema).max(MAX_CONNECTION_HISTORY_OBSERVATIONS)
|
|
31
|
+
}).strict();
|
|
32
|
+
|
|
33
|
+
export { CONNECTION_HISTORY_RETENTION_MS, MAX_CONNECTION_HISTORY_BATCH, MAX_CONNECTION_HISTORY_OBSERVATIONS, connectionHistoryGrantSchema, connectionHistoryObservationSchema, connectionHistoryResponseSchema, connectionHistoryUploadSchema };
|
|
@@ -1,3 +1,4 @@
|
|
|
1
|
+
import { connectionHistoryGrantSchema } from './chunk-MCIXU6XP.js';
|
|
1
2
|
import { z } from 'zod';
|
|
2
3
|
|
|
3
4
|
// ../http-client/src/errors.ts
|
|
@@ -214,6 +215,44 @@ var schema26 = z.strictObject({
|
|
|
214
215
|
});
|
|
215
216
|
var clipLibraryResponseSchema = schema24;
|
|
216
217
|
var clipLibraryUpdateSchema = schema26;
|
|
218
|
+
var RECORDED_MEDIA_MODES = ["audio", "video", "audio_video"];
|
|
219
|
+
var recordingModeSchema = z.enum(["off", ...RECORDED_MEDIA_MODES]).describe("Server-owned recording policy; omitted means off. Audio includes user and avatar audio.");
|
|
220
|
+
var RECORDING_STATUSES = ["pending", "recording", "processing", "ready", "failed", "expired"];
|
|
221
|
+
z.enum(RECORDING_STATUSES);
|
|
222
|
+
var recordingMetadataSchema = z.object({
|
|
223
|
+
sessionId: z.string().min(1),
|
|
224
|
+
recordingId: z.string().min(1),
|
|
225
|
+
mode: recordingModeSchema.exclude(["off"]),
|
|
226
|
+
createdAt: z.string().datetime({ offset: true }),
|
|
227
|
+
retainedUntil: z.string().datetime({ offset: true })
|
|
228
|
+
});
|
|
229
|
+
var recordingArtifactSchema = z.discriminatedUnion("status", [
|
|
230
|
+
recordingMetadataSchema.extend({ status: z.enum(["pending", "recording", "processing", "expired"]) }).strict(),
|
|
231
|
+
recordingMetadataSchema.extend({
|
|
232
|
+
status: z.literal("ready"),
|
|
233
|
+
mediaType: z.enum(["audio/mp4", "video/mp4"]),
|
|
234
|
+
sizeBytes: z.number().int().positive(),
|
|
235
|
+
durationMs: z.number().int().nonnegative().nullable()
|
|
236
|
+
}).strict(),
|
|
237
|
+
recordingMetadataSchema.extend({
|
|
238
|
+
status: z.literal("failed"),
|
|
239
|
+
errorCode: z.enum(["recording_failed", "recording_unavailable"])
|
|
240
|
+
}).strict()
|
|
241
|
+
]);
|
|
242
|
+
var listRecordingsQuerySchema = z.object({
|
|
243
|
+
sessionId: z.string().min(1).optional(),
|
|
244
|
+
limit: z.coerce.number().int().min(1).max(100).default(50),
|
|
245
|
+
cursor: z.string().min(1).optional()
|
|
246
|
+
}).strict();
|
|
247
|
+
var listRecordingsResponseSchema = z.object({
|
|
248
|
+
data: z.array(recordingArtifactSchema),
|
|
249
|
+
nextCursor: z.string().nullable()
|
|
250
|
+
}).strict();
|
|
251
|
+
var recordingAccessResponseSchema = z.object({
|
|
252
|
+
recordingId: z.string().min(1),
|
|
253
|
+
url: z.string().url(),
|
|
254
|
+
expiresAt: z.string().datetime({ offset: true })
|
|
255
|
+
}).strict();
|
|
217
256
|
|
|
218
257
|
// ../http-client/src/retry.ts
|
|
219
258
|
var RETRYABLE_STATUS = /* @__PURE__ */ new Set([408, 500, 502, 503, 504]);
|
|
@@ -233,10 +272,335 @@ function isTransient(cause) {
|
|
|
233
272
|
const name = cause?.name;
|
|
234
273
|
return name === "TimeoutError" || name === "TypeError" || name === "FetchError";
|
|
235
274
|
}
|
|
275
|
+
var DEFAULT_AVATAR_ID = "maria";
|
|
276
|
+
var DEFAULT_BACKGROUND_ID = "plain_white";
|
|
277
|
+
var sessionModeSchema = z.enum(["avatar", "voice"]);
|
|
278
|
+
var DEFAULT_SESSION_MODE = "avatar";
|
|
279
|
+
var DEFAULT_MAX_SESSION_SECONDS = 1800;
|
|
280
|
+
var avatarSourceKindSchema = z.enum(["portrait", "source_video"]);
|
|
281
|
+
var liveKitSttModeSchema = z.enum(["server", "off"]);
|
|
282
|
+
var renderBackendSchema = z.string();
|
|
283
|
+
var sessionLiveEditSchema = z.object({
|
|
284
|
+
rules: z.string().min(1).max(2e3),
|
|
285
|
+
cooldown_seconds: z.number().int().min(5).max(600).optional(),
|
|
286
|
+
// Which machinery runs the re-edit. Absent ⇒ the server default ("editor"). A deploy
|
|
287
|
+
// that cannot provide the requested renderer serves the editor lane and logs it,
|
|
288
|
+
// rather than failing a call that would otherwise have connected fine.
|
|
289
|
+
renderer: z.enum(["editor", "generative"]).optional()
|
|
290
|
+
}).strict();
|
|
291
|
+
var sessionSupportEditsSchema = z.object({
|
|
292
|
+
instruction: z.string().min(1).max(1e3),
|
|
293
|
+
reference_url: z.string().url().optional(),
|
|
294
|
+
live_edit: sessionLiveEditSchema.optional()
|
|
295
|
+
}).strict();
|
|
296
|
+
var LLM_PROVIDERS = ["local", "gemini", "openai"];
|
|
297
|
+
var llmProviderSchema = z.enum(LLM_PROVIDERS);
|
|
298
|
+
var llmConfigSchema = z.object({
|
|
299
|
+
backend: llmProviderSchema.optional(),
|
|
300
|
+
model: z.string().max(200).nullable().optional()
|
|
301
|
+
}).strict();
|
|
302
|
+
var llmSelectionSchema = z.object({
|
|
303
|
+
provider: llmProviderSchema,
|
|
304
|
+
model: z.string().max(200).nullable().optional()
|
|
305
|
+
}).strict();
|
|
306
|
+
var liveKitInitialContextMessageSchema = z.object({
|
|
307
|
+
role: z.enum(["system", "user", "assistant"]),
|
|
308
|
+
content: z.string().min(1).max(4e3)
|
|
309
|
+
}).strict();
|
|
310
|
+
var CARTESIA_TTS_MODELS = [
|
|
311
|
+
"cartesia/sonic-2",
|
|
312
|
+
"cartesia/sonic-2-latest",
|
|
313
|
+
"cartesia/sonic-3",
|
|
314
|
+
"cartesia/sonic-3-latest",
|
|
315
|
+
"cartesia/sonic-turbo",
|
|
316
|
+
"cartesia/sonic-turbo-latest"
|
|
317
|
+
];
|
|
318
|
+
var cartesiaTtsModelSchema = z.enum(CARTESIA_TTS_MODELS);
|
|
319
|
+
var cartesiaVoiceSpecSchema = z.object({
|
|
320
|
+
provider: z.literal("cartesia"),
|
|
321
|
+
model: cartesiaTtsModelSchema.default("cartesia/sonic-3"),
|
|
322
|
+
voice_id: z.string().min(1).max(120),
|
|
323
|
+
speed: z.number().min(0.5).max(2).nullable().optional(),
|
|
324
|
+
emotion: z.string().min(1).max(80).nullable().optional(),
|
|
325
|
+
language: z.string().min(2).max(16).nullable().optional()
|
|
326
|
+
}).strict();
|
|
327
|
+
var FISH_TTS_MODELS = ["speech-1.6", "s1", "s2-pro", "speech-1.5", "s1-mini"];
|
|
328
|
+
var fishTtsModelSchema = z.enum(FISH_TTS_MODELS);
|
|
329
|
+
var breezeVoiceSpecSchema = z.object({
|
|
330
|
+
provider: z.literal("breezeblue"),
|
|
331
|
+
model: z.string().min(1).max(80).default("bluebell-v1-en"),
|
|
332
|
+
voice_id: z.string().min(1).max(120),
|
|
333
|
+
guidance_scale: z.number().min(1).max(10).nullable().optional(),
|
|
334
|
+
instructions: z.string().min(1).max(1e3).nullable().optional(),
|
|
335
|
+
language: z.string().min(2).max(16).nullable().optional()
|
|
336
|
+
}).strict();
|
|
337
|
+
var fishVoiceSpecSchema = z.object({
|
|
338
|
+
provider: z.literal("fish"),
|
|
339
|
+
model: fishTtsModelSchema.default("speech-1.6"),
|
|
340
|
+
voice_id: z.string().min(1).max(120),
|
|
341
|
+
speed: z.number().min(0.5).max(2).nullable().optional(),
|
|
342
|
+
emotion: z.string().min(1).max(80).nullable().optional(),
|
|
343
|
+
language: z.string().min(2).max(16).nullable().optional()
|
|
344
|
+
}).strict();
|
|
345
|
+
var voiceSpecSchema = z.discriminatedUnion("provider", [
|
|
346
|
+
cartesiaVoiceSpecSchema,
|
|
347
|
+
breezeVoiceSpecSchema,
|
|
348
|
+
fishVoiceSpecSchema
|
|
349
|
+
]);
|
|
350
|
+
var nullableUrlSchema = z.string().url().nullable();
|
|
351
|
+
var clipTriggerSchema = z.enum([
|
|
352
|
+
"idle",
|
|
353
|
+
"listen",
|
|
354
|
+
"think",
|
|
355
|
+
"directive"
|
|
356
|
+
]);
|
|
357
|
+
var sessionClipSchema = z.object({
|
|
358
|
+
clip_id: z.string().regex(/^[A-Za-z0-9_-]{1,64}$/, "clip_id must be a slug").refine((id) => id !== "primary", "'primary' is reserved for the avatar's source video"),
|
|
359
|
+
source_video_url: z.string().url().optional(),
|
|
360
|
+
video_cache_id: z.string().min(8).max(160).optional(),
|
|
361
|
+
max_seconds: z.number().min(1).max(10).optional(),
|
|
362
|
+
trigger: clipTriggerSchema.optional(),
|
|
363
|
+
loop: z.boolean().optional(),
|
|
364
|
+
weight: z.number().min(0).max(100).optional(),
|
|
365
|
+
crossfade_ms: z.number().int().min(0).max(1e3).optional(),
|
|
366
|
+
trim_start_ms: z.number().int().min(0).max(2e3).optional(),
|
|
367
|
+
trim_end_ms: z.number().int().min(0).max(2e3).optional(),
|
|
368
|
+
// The cue the character reads to decide this clip. `when` is the public name and the
|
|
369
|
+
// one the docs use; `hint` is the name the wire first shipped under and still accepts.
|
|
370
|
+
// Both are listed because this object is `.strict()` — omitting `when` would make the
|
|
371
|
+
// public name a validation error. Send one, never both.
|
|
372
|
+
when: z.string().min(1).max(120).optional(),
|
|
373
|
+
hint: z.string().min(1).max(120).optional()
|
|
374
|
+
}).strict().refine((clip) => clip.source_video_url || clip.video_cache_id, {
|
|
375
|
+
message: "a clip needs source_video_url or video_cache_id"
|
|
376
|
+
}).refine((clip) => !(clip.when && clip.hint), {
|
|
377
|
+
message: "set `when` or `hint`, not both \u2014 they are the same field",
|
|
378
|
+
path: ["when"]
|
|
379
|
+
});
|
|
380
|
+
var sessionChoreographySchema = z.object({
|
|
381
|
+
idle_dwell_min_seconds: z.number().min(1).max(60).optional(),
|
|
382
|
+
idle_dwell_max_seconds: z.number().min(1).max(120).optional(),
|
|
383
|
+
special_weight: z.number().min(0).max(100).optional(),
|
|
384
|
+
start_grace_seconds: z.number().min(0).max(60).optional(),
|
|
385
|
+
crossfade_ms: z.number().int().min(0).max(1e3).optional(),
|
|
386
|
+
crossfade_easing: z.enum(["linear", "smooth", "ease_out"]).optional(),
|
|
387
|
+
wrap_crossfade_ms: z.number().int().min(0).max(1e3).optional()
|
|
388
|
+
}).strict().refine(
|
|
389
|
+
(c) => c.idle_dwell_min_seconds === void 0 || c.idle_dwell_max_seconds === void 0 || c.idle_dwell_min_seconds <= c.idle_dwell_max_seconds,
|
|
390
|
+
{ message: "idle_dwell_min_seconds must be <= idle_dwell_max_seconds" }
|
|
391
|
+
);
|
|
392
|
+
var sessionBehaviorSchema = z.object({
|
|
393
|
+
gestures_enabled: z.boolean().optional()
|
|
394
|
+
}).strict();
|
|
395
|
+
var transcriptWebhookSchema = z.object({
|
|
396
|
+
url: z.string().url().max(500),
|
|
397
|
+
secret: z.string().min(16).max(200)
|
|
398
|
+
}).strict();
|
|
399
|
+
var clientMetadataSchema = z.record(z.string().min(1).max(64), z.string().max(200)).refine((value) => Object.keys(value).length <= 16, {
|
|
400
|
+
message: "client_metadata carries at most 16 entries"
|
|
401
|
+
});
|
|
402
|
+
var MAX_SESSION_INSTRUCTIONS_CHARS = 8e3;
|
|
403
|
+
z.object({
|
|
404
|
+
avatar_id: z.string().min(1).max(160).default(DEFAULT_AVATAR_ID),
|
|
405
|
+
background_id: z.string().min(1).max(160).default(DEFAULT_BACKGROUND_ID),
|
|
406
|
+
mode: sessionModeSchema.default(DEFAULT_SESSION_MODE),
|
|
407
|
+
create_room: z.boolean().default(true),
|
|
408
|
+
dispatch_agent: z.boolean().default(true),
|
|
409
|
+
instructions: z.string().min(1).max(MAX_SESSION_INSTRUCTIONS_CHARS).optional(),
|
|
410
|
+
initial_context: z.array(liveKitInitialContextMessageSchema).max(32).default([]),
|
|
411
|
+
initial_say: z.string().min(1).max(1e3).optional(),
|
|
412
|
+
llm: llmConfigSchema.nullable().optional(),
|
|
413
|
+
max_session_seconds: z.number().int().min(1).max(DEFAULT_MAX_SESSION_SECONDS).optional(),
|
|
414
|
+
participant_identity: z.string().min(1).max(160).optional(),
|
|
415
|
+
participant_name: z.string().max(160).optional(),
|
|
416
|
+
queue_ticket_id: z.string().min(1).max(160).optional(),
|
|
417
|
+
portrait_url: nullableUrlSchema.optional(),
|
|
418
|
+
room_name: z.string().min(1).max(160).optional(),
|
|
419
|
+
source_kind: avatarSourceKindSchema.default("portrait"),
|
|
420
|
+
source_video_url: nullableUrlSchema.optional(),
|
|
421
|
+
stt_mode: liveKitSttModeSchema.default("server"),
|
|
422
|
+
video_cache_id: z.string().min(1).max(240).nullable().optional(),
|
|
423
|
+
voice: voiceSpecSchema.nullable().optional(),
|
|
424
|
+
voice_id: z.string().min(1).max(240).nullable().optional(),
|
|
425
|
+
// Deliberately UNBOUNDED. This object is `.strict()`, so it rejects rather than trims:
|
|
426
|
+
// a count cap here does not mean "use fewer clips", it means "there is no call". How
|
|
427
|
+
// many a session actually warms is decided where the clips are loaded, and loading
|
|
428
|
+
// fewer is always safe — so the wire must not hold a number too.
|
|
429
|
+
clip_library: z.array(sessionClipSchema).optional(),
|
|
430
|
+
choreography: sessionChoreographySchema.optional(),
|
|
431
|
+
behavior: sessionBehaviorSchema.optional(),
|
|
432
|
+
expression_profile: z.string().min(1).max(40).optional(),
|
|
433
|
+
render_backend: renderBackendSchema.optional(),
|
|
434
|
+
support_edits: sessionSupportEditsSchema.optional(),
|
|
435
|
+
transcript_webhook: transcriptWebhookSchema.optional(),
|
|
436
|
+
client_metadata: clientMetadataSchema.optional(),
|
|
437
|
+
connection_history: z.boolean().optional()
|
|
438
|
+
}).strict().superRefine((value, ctx) => {
|
|
439
|
+
if (value.support_edits && value.render_backend === "generative") {
|
|
440
|
+
ctx.addIssue({
|
|
441
|
+
code: "custom",
|
|
442
|
+
message: "support_edits needs a source video to edit; it cannot be combined with render_backend='generative'",
|
|
443
|
+
path: ["support_edits"]
|
|
444
|
+
});
|
|
445
|
+
}
|
|
446
|
+
if (value.support_edits && value.mode === "voice") {
|
|
447
|
+
ctx.addIssue({
|
|
448
|
+
code: "custom",
|
|
449
|
+
message: "support_edits needs a video session; it cannot be combined with mode='voice'",
|
|
450
|
+
path: ["support_edits"]
|
|
451
|
+
});
|
|
452
|
+
}
|
|
453
|
+
if (value.source_kind === "portrait") {
|
|
454
|
+
if (value.source_video_url || value.video_cache_id) {
|
|
455
|
+
ctx.addIssue({
|
|
456
|
+
code: "custom",
|
|
457
|
+
message: "source_video_url/video_cache_id require source_kind='source_video'",
|
|
458
|
+
path: ["source_kind"]
|
|
459
|
+
});
|
|
460
|
+
}
|
|
461
|
+
return;
|
|
462
|
+
}
|
|
463
|
+
if (value.portrait_url) {
|
|
464
|
+
ctx.addIssue({
|
|
465
|
+
code: "custom",
|
|
466
|
+
message: "portrait_url cannot be combined with source_kind='source_video'",
|
|
467
|
+
path: ["portrait_url"]
|
|
468
|
+
});
|
|
469
|
+
}
|
|
470
|
+
if (!value.source_video_url && !value.video_cache_id) {
|
|
471
|
+
ctx.addIssue({
|
|
472
|
+
code: "custom",
|
|
473
|
+
message: "source_kind='source_video' requires source_video_url or video_cache_id",
|
|
474
|
+
path: ["source_video_url"]
|
|
475
|
+
});
|
|
476
|
+
}
|
|
477
|
+
});
|
|
478
|
+
z.object({
|
|
479
|
+
avatarId: z.string().min(1).max(160),
|
|
480
|
+
backgroundId: z.string().min(1).max(160).default(DEFAULT_BACKGROUND_ID),
|
|
481
|
+
mode: sessionModeSchema.default(DEFAULT_SESSION_MODE),
|
|
482
|
+
createRoom: z.boolean().default(true),
|
|
483
|
+
dispatchAgent: z.boolean().default(true),
|
|
484
|
+
instructions: z.string().min(1).max(MAX_SESSION_INSTRUCTIONS_CHARS).optional(),
|
|
485
|
+
initialContext: z.array(liveKitInitialContextMessageSchema).max(32).default([]),
|
|
486
|
+
initialSay: z.string().min(1).max(1e3).optional(),
|
|
487
|
+
llm: llmSelectionSchema.nullable().optional(),
|
|
488
|
+
maxSessionSeconds: z.number().int().min(1).max(DEFAULT_MAX_SESSION_SECONDS).optional(),
|
|
489
|
+
participantIdentity: z.string().min(1).max(160).optional(),
|
|
490
|
+
participantName: z.string().max(160).optional(),
|
|
491
|
+
queueTicketId: z.string().min(1).max(160).optional(),
|
|
492
|
+
roomName: z.string().min(1).max(160).optional(),
|
|
493
|
+
sttMode: liveKitSttModeSchema.default("server"),
|
|
494
|
+
voice: voiceSpecSchema.nullable().optional(),
|
|
495
|
+
voiceId: z.string().min(1).max(240).nullable().optional(),
|
|
496
|
+
// Unbounded, for the same reason as `clip_library` on the wire schema above.
|
|
497
|
+
clipLibrary: z.array(sessionClipSchema).optional(),
|
|
498
|
+
behavior: sessionBehaviorSchema.optional(),
|
|
499
|
+
renderBackend: renderBackendSchema.optional(),
|
|
500
|
+
supportEdits: sessionSupportEditsSchema.optional(),
|
|
501
|
+
transcriptWebhook: transcriptWebhookSchema.optional(),
|
|
502
|
+
clientMetadata: clientMetadataSchema.optional(),
|
|
503
|
+
connectionHistory: z.boolean().optional()
|
|
504
|
+
}).strict();
|
|
505
|
+
var liveKitSessionGrantSchema = z.object({
|
|
506
|
+
recording: recordingArtifactSchema.optional(),
|
|
507
|
+
connection_history: connectionHistoryGrantSchema.optional(),
|
|
508
|
+
status: z.literal("ready").default("ready"),
|
|
509
|
+
session_id: z.string().min(1),
|
|
510
|
+
room_name: z.string().min(1),
|
|
511
|
+
livekit_url: z.string().min(1),
|
|
512
|
+
participant_token: z.string().min(1),
|
|
513
|
+
participant_identity: z.string().min(1),
|
|
514
|
+
reservation_expires_at: z.string().datetime({ offset: true }),
|
|
515
|
+
stt_mode: liveKitSttModeSchema.default("server"),
|
|
516
|
+
room_created: z.boolean().default(false),
|
|
517
|
+
dispatch_created: z.boolean().default(false),
|
|
518
|
+
join_timeout_seconds: z.number().int().nonnegative().default(0),
|
|
519
|
+
idle_timeout_seconds: z.number().int().nonnegative().default(0),
|
|
520
|
+
max_session_seconds: z.number().int().nonnegative().default(0)
|
|
521
|
+
}).passthrough();
|
|
522
|
+
var sessionEndReasonSchema = z.enum([
|
|
523
|
+
"user_ended",
|
|
524
|
+
"session_cap",
|
|
525
|
+
"idle",
|
|
526
|
+
"disconnected",
|
|
527
|
+
"out_of_credits",
|
|
528
|
+
"agent_ended",
|
|
529
|
+
"failed"
|
|
530
|
+
]);
|
|
531
|
+
var approachingEndReasonSchema = z.enum(["session_cap", "idle"]);
|
|
532
|
+
var sessionClockFrameSchema = z.object({
|
|
533
|
+
kind: z.literal("session_clock"),
|
|
534
|
+
started_at_unix_ms: z.number().int().nonnegative(),
|
|
535
|
+
max_session_seconds: z.number().int().nonnegative(),
|
|
536
|
+
idle_timeout_seconds: z.number().int().nonnegative()
|
|
537
|
+
}).strict();
|
|
538
|
+
var endingFrameSchema = z.object({ kind: z.literal("ending"), reason: approachingEndReasonSchema }).strict();
|
|
539
|
+
var closingTurnDoneFrameSchema = z.object({ kind: z.literal("closing_turn_done"), turn_id: z.string().min(1) }).strict();
|
|
540
|
+
var endedFrameSchema = z.object({ kind: z.literal("ended"), reason: sessionEndReasonSchema }).strict();
|
|
541
|
+
var behaviorStateFrameSchema = z.object({
|
|
542
|
+
kind: z.literal("behavior_state"),
|
|
543
|
+
state: z.string().min(1).max(32),
|
|
544
|
+
clip_id: z.string().min(1).max(64).optional(),
|
|
545
|
+
trigger: clipTriggerSchema.optional(),
|
|
546
|
+
loop: z.boolean().optional(),
|
|
547
|
+
prev_clip_id: z.string().min(1).max(64).optional()
|
|
548
|
+
}).strip();
|
|
549
|
+
var clipAckFrameSchema = z.object({
|
|
550
|
+
kind: z.literal("clip_ack"),
|
|
551
|
+
request_id: z.string().max(64),
|
|
552
|
+
accepted: z.boolean(),
|
|
553
|
+
reason: z.string().max(64)
|
|
554
|
+
}).strip();
|
|
555
|
+
z.discriminatedUnion("kind", [
|
|
556
|
+
sessionClockFrameSchema,
|
|
557
|
+
endingFrameSchema,
|
|
558
|
+
closingTurnDoneFrameSchema,
|
|
559
|
+
endedFrameSchema,
|
|
560
|
+
behaviorStateFrameSchema,
|
|
561
|
+
clipAckFrameSchema
|
|
562
|
+
]);
|
|
563
|
+
var liveKitCapacitySnapshotSchema = z.object({
|
|
564
|
+
// Placement identity + per-worker session ceiling. These are always present on the
|
|
565
|
+
// wire (the platform serializes them and they come back on the grant, so a customer
|
|
566
|
+
// already sees them) but are typed OPTIONAL here on purpose: this SDK is the defensive
|
|
567
|
+
// READER of that wire, so a consumer must tolerate a response variant that omits them
|
|
568
|
+
// rather than hard-fail parse. `max_sessions_per_gpu` is how many sessions one waking
|
|
569
|
+
// worker serves — a queue-depth estimate input a consumer reads off the busy response.
|
|
570
|
+
capacity_pool: z.string().min(1).optional(),
|
|
571
|
+
agent_name: z.string().min(1).optional(),
|
|
572
|
+
max_sessions: z.number().int().nonnegative(),
|
|
573
|
+
max_sessions_per_gpu: z.number().int().positive().optional(),
|
|
574
|
+
worker_count: z.number().int().nonnegative(),
|
|
575
|
+
active_sessions: z.number().int().nonnegative(),
|
|
576
|
+
reserved_sessions: z.number().int().nonnegative(),
|
|
577
|
+
observed_worker_active_sessions: z.number().int().nonnegative(),
|
|
578
|
+
available_sessions: z.number().int().nonnegative(),
|
|
579
|
+
queue_size: z.number().int().nonnegative(),
|
|
580
|
+
admission_open: z.boolean(),
|
|
581
|
+
recommended_retry_ms: z.number().int().nonnegative(),
|
|
582
|
+
load: z.number().min(0).max(1)
|
|
583
|
+
}).passthrough();
|
|
584
|
+
z.object({
|
|
585
|
+
message: z.string().min(1),
|
|
586
|
+
capacity: liveKitCapacitySnapshotSchema,
|
|
587
|
+
queue_size: z.number().int().nonnegative(),
|
|
588
|
+
queue_ticket_id: z.string().min(1).optional(),
|
|
589
|
+
queue_position: z.number().int().positive().optional(),
|
|
590
|
+
recommended_retry_ms: z.number().int().nonnegative()
|
|
591
|
+
}).strict();
|
|
592
|
+
z.enum([
|
|
593
|
+
"page_hide",
|
|
594
|
+
"disconnected",
|
|
595
|
+
"superseded",
|
|
596
|
+
"unmount",
|
|
597
|
+
"manual",
|
|
598
|
+
"idle_timeout"
|
|
599
|
+
]);
|
|
236
600
|
|
|
237
601
|
// ../http-client/src/client.ts
|
|
238
602
|
var DEFAULT_BASE_URL = "https://realtimeavatar.ai/api/v1";
|
|
239
|
-
var SDK_VERSION = "0.
|
|
603
|
+
var SDK_VERSION = "0.17.0";
|
|
240
604
|
var RealtimeAvatar = class {
|
|
241
605
|
#apiKey;
|
|
242
606
|
#baseUrl;
|
|
@@ -281,6 +645,8 @@ var RealtimeAvatar = class {
|
|
|
281
645
|
if (options.maxSeconds !== void 0) body.max_session_seconds = Math.floor(options.maxSeconds);
|
|
282
646
|
if (options.voice !== void 0) body.voice = options.voice;
|
|
283
647
|
if (options.metadata !== void 0) body.client_metadata = options.metadata;
|
|
648
|
+
if (options.recording !== void 0) body.recording = recordingModeSchema.parse(options.recording);
|
|
649
|
+
if (options.connectionHistory !== void 0) body.connection_history = options.connectionHistory;
|
|
284
650
|
if (options.clientTools) body.capabilities = ["client_tools"];
|
|
285
651
|
if (options.transcript !== void 0) {
|
|
286
652
|
body.transcript_webhook = { url: options.transcript.url, secret: options.transcript.secret };
|
|
@@ -299,20 +665,24 @@ var RealtimeAvatar = class {
|
|
|
299
665
|
};
|
|
300
666
|
}
|
|
301
667
|
}
|
|
302
|
-
const
|
|
668
|
+
const raw = await this.#json(response);
|
|
669
|
+
const grant = liveKitSessionGrantSchema.parse(raw);
|
|
670
|
+
if (!isRecord(raw)) throw new RealtimeAvatarError("Invalid session grant");
|
|
303
671
|
return {
|
|
304
672
|
status: "ready",
|
|
305
|
-
|
|
306
|
-
|
|
307
|
-
|
|
308
|
-
|
|
309
|
-
|
|
310
|
-
|
|
311
|
-
|
|
312
|
-
|
|
673
|
+
...grant.recording === void 0 ? {} : { recording: grant.recording },
|
|
674
|
+
sessionId: grant.session_id,
|
|
675
|
+
roomName: grant.room_name,
|
|
676
|
+
livekitUrl: grant.livekit_url,
|
|
677
|
+
participantToken: grant.participant_token,
|
|
678
|
+
participantIdentity: grant.participant_identity,
|
|
679
|
+
maxSessionSeconds: grant.max_session_seconds,
|
|
680
|
+
idleTimeoutSeconds: grant.idle_timeout_seconds,
|
|
681
|
+
reservationExpiresAt: grant.reservation_expires_at,
|
|
313
682
|
// The parsed fields above are for YOUR logic. Relay `raw` to the client untouched:
|
|
314
|
-
// the browser SDK
|
|
315
|
-
|
|
683
|
+
// the browser SDK accepts provider-issued capability fields through its passthrough
|
|
684
|
+
// grant schema; forwarding verbatim preserves capabilities this wrapper does not model.
|
|
685
|
+
raw
|
|
316
686
|
};
|
|
317
687
|
}
|
|
318
688
|
/**
|
|
@@ -632,6 +1002,30 @@ var RealtimeAvatar = class {
|
|
|
632
1002
|
cursor = page.nextCursor ?? void 0;
|
|
633
1003
|
} while (cursor);
|
|
634
1004
|
}
|
|
1005
|
+
/** Recording metadata for this account, optionally limited to one call. Requires recordings:read. */
|
|
1006
|
+
async listRecordings(options = {}) {
|
|
1007
|
+
const parsed = listRecordingsQuerySchema.parse(options);
|
|
1008
|
+
const query = new URLSearchParams();
|
|
1009
|
+
if (parsed.sessionId !== void 0) query.set("sessionId", parsed.sessionId);
|
|
1010
|
+
if (options.limit !== void 0) query.set("limit", String(parsed.limit));
|
|
1011
|
+
if (parsed.cursor !== void 0) query.set("cursor", parsed.cursor);
|
|
1012
|
+
const suffix = query.size ? `?${query}` : "";
|
|
1013
|
+
return listRecordingsResponseSchema.parse(await this.#json(await this.#request("GET", `/recordings${suffix}`)));
|
|
1014
|
+
}
|
|
1015
|
+
/** Current recording status. Final media can become ready after the call has ended. */
|
|
1016
|
+
async getRecording(recordingId) {
|
|
1017
|
+
return recordingArtifactSchema.parse(await this.#json(await this.#request("GET", `/recordings/${encodeURIComponent(recordingId)}`)));
|
|
1018
|
+
}
|
|
1019
|
+
/** Renewable playback access. Keep recordingId in storage; URLs expire at expiresAt. */
|
|
1020
|
+
async getRecordingAccess(recordingId) {
|
|
1021
|
+
return recordingAccessResponseSchema.parse(await this.#json(await this.#request("GET", `/recordings/${encodeURIComponent(recordingId)}/access`)));
|
|
1022
|
+
}
|
|
1023
|
+
/** Read one session's bounded LiveKit connection history. Requires `usage:read`. */
|
|
1024
|
+
async getConnectionHistory(sessionId) {
|
|
1025
|
+
if (!sessionId) throw new RealtimeAvatarError("sessionId is required");
|
|
1026
|
+
const { connectionHistoryResponseSchema } = await import('./connection-history-CXL5QE56.js');
|
|
1027
|
+
return connectionHistoryResponseSchema.parse(await this.#json(await this.#request("GET", `/sessions/${encodeURIComponent(sessionId)}/connection-history`)));
|
|
1028
|
+
}
|
|
635
1029
|
async creditBalance() {
|
|
636
1030
|
return await this.#json(await this.#request("GET", "/credits/balance"));
|
|
637
1031
|
}
|
|
@@ -811,4 +1205,4 @@ function isQueued(result) {
|
|
|
811
1205
|
return "queued" in result;
|
|
812
1206
|
}
|
|
813
1207
|
|
|
814
|
-
export { RealtimeAvatar, RealtimeAvatarError, RealtimeAvatarHttpError, clipLibraryDeclarationSchema, isQueued, verifyTranscript };
|
|
1208
|
+
export { RealtimeAvatar, RealtimeAvatarError, RealtimeAvatarHttpError, clipLibraryDeclarationSchema, isQueued, listRecordingsResponseSchema, recordingAccessResponseSchema, recordingArtifactSchema, recordingModeSchema, verifyTranscript };
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
export { CONNECTION_HISTORY_RETENTION_MS, MAX_CONNECTION_HISTORY_BATCH, MAX_CONNECTION_HISTORY_OBSERVATIONS, connectionHistoryGrantSchema, connectionHistoryObservationSchema, connectionHistoryResponseSchema, connectionHistoryUploadSchema } from './chunk-MCIXU6XP.js';
|