vsip-sdk 0.1.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/LICENSE +21 -0
- package/README.md +70 -0
- package/dist/client.d.ts +50 -0
- package/dist/client.js +163 -0
- package/dist/events.d.ts +93 -0
- package/dist/events.js +14 -0
- package/dist/index.d.ts +6 -0
- package/dist/index.js +3 -0
- package/dist/session.d.ts +57 -0
- package/dist/session.js +179 -0
- package/package.json +31 -0
package/LICENSE
ADDED
|
@@ -0,0 +1,21 @@
|
|
|
1
|
+
MIT License
|
|
2
|
+
|
|
3
|
+
Copyright (c) 2026 VSIP
|
|
4
|
+
|
|
5
|
+
Permission is hereby granted, free of charge, to any person obtaining a copy
|
|
6
|
+
of this software and associated documentation files (the "Software"), to deal
|
|
7
|
+
in the Software without restriction, including without limitation the rights
|
|
8
|
+
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
|
9
|
+
copies of the Software, and to permit persons to whom the Software is
|
|
10
|
+
furnished to do so, subject to the following conditions:
|
|
11
|
+
|
|
12
|
+
The above copyright notice and this permission notice shall be included in all
|
|
13
|
+
copies or substantial portions of the Software.
|
|
14
|
+
|
|
15
|
+
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
|
16
|
+
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
|
17
|
+
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
|
18
|
+
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
|
19
|
+
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
|
20
|
+
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
|
|
21
|
+
SOFTWARE.
|
package/README.md
ADDED
|
@@ -0,0 +1,70 @@
|
|
|
1
|
+
# vsip-sdk
|
|
2
|
+
|
|
3
|
+
JavaScript / TypeScript SDK for the VSIP Voice Events API — real-time turn
|
|
4
|
+
detection, identity-gated barge-in, and speaker verification. Runs in the
|
|
5
|
+
browser and Node 18+.
|
|
6
|
+
|
|
7
|
+
```bash
|
|
8
|
+
npm install vsip-sdk
|
|
9
|
+
```
|
|
10
|
+
|
|
11
|
+
## Browser quickstart
|
|
12
|
+
|
|
13
|
+
```ts
|
|
14
|
+
import { VSIPClient, VoiceSession, type Turn } from "vsip-sdk";
|
|
15
|
+
|
|
16
|
+
const client = new VSIPClient({
|
|
17
|
+
apiKey: "vsip_...", // browser: sent as ?token= (proxy in prod)
|
|
18
|
+
url: "wss://api.vsip.io/v1/stream",
|
|
19
|
+
});
|
|
20
|
+
|
|
21
|
+
const session = new VoiceSession(client, {
|
|
22
|
+
onTurn: async (turn: Turn) => {
|
|
23
|
+
if (!turn.shouldRespond) return; // verified: not your enrolled user
|
|
24
|
+
const text = await myStt(turn.audio); // Int16Array PCM
|
|
25
|
+
const reply = await myLlm(text);
|
|
26
|
+
await session.speakGate(); // don't talk over a turn
|
|
27
|
+
client.agentSpeaking();
|
|
28
|
+
await myTtsPlay(reply); // stop this in onBargeIn
|
|
29
|
+
client.agentIdle();
|
|
30
|
+
},
|
|
31
|
+
onBargeIn: () => myTtsStop(),
|
|
32
|
+
});
|
|
33
|
+
|
|
34
|
+
await client.connect();
|
|
35
|
+
session.start();
|
|
36
|
+
|
|
37
|
+
// Feed 16kHz mono Int16 PCM frames from the Web Audio API:
|
|
38
|
+
scriptProcessor.onaudioprocess = (e) => {
|
|
39
|
+
const f32 = e.inputBuffer.getChannelData(0);
|
|
40
|
+
const i16 = new Int16Array(f32.length);
|
|
41
|
+
for (let i = 0; i < f32.length; i++) i16[i] = Math.max(-1, Math.min(1, f32[i])) * 0x7fff;
|
|
42
|
+
session.sendAudio(i16);
|
|
43
|
+
};
|
|
44
|
+
```
|
|
45
|
+
|
|
46
|
+
## What `VoiceSession` handles for you
|
|
47
|
+
|
|
48
|
+
Same field-learned patterns as the Python SDK: onset-lookback turn slicing
|
|
49
|
+
(1.0s / 1.5s for barge-ins), pre-lock fast path + **post-lock
|
|
50
|
+
`speaker_verification` gating** (turns held until the verdict, so an impostor
|
|
51
|
+
is never answered), `speakGate()`, queued turn dispatch, and a capped idle
|
|
52
|
+
buffer. Gate your agent on `turn.shouldRespond`.
|
|
53
|
+
|
|
54
|
+
`VSIPClient` alone gives you the typed event stream (`client.on(handler)`),
|
|
55
|
+
control commands (`agentSpeaking`/`agentIdle`, `setBargeInMode`, `toggleLock`,
|
|
56
|
+
`reset`), and transparent reconnect + resume.
|
|
57
|
+
|
|
58
|
+
## Node usage
|
|
59
|
+
|
|
60
|
+
Node 22 has a global `WebSocket`. On Node 18–20, pass one:
|
|
61
|
+
|
|
62
|
+
```ts
|
|
63
|
+
import WebSocket from "ws";
|
|
64
|
+
const client = new VSIPClient({ apiKey, url, webSocketImpl: WebSocket as never });
|
|
65
|
+
```
|
|
66
|
+
|
|
67
|
+
## Types & compatibility
|
|
68
|
+
|
|
69
|
+
Fully typed (discriminated `VSIPEvent` union). Unknown event types and fields
|
|
70
|
+
parse without error, so this SDK survives additive protocol growth.
|
package/dist/client.d.ts
ADDED
|
@@ -0,0 +1,50 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* VSIPClient — WebSocket client for the VSIP Voice Events API.
|
|
3
|
+
* Browser (native WebSocket) and Node 18+ (global WebSocket / injected impl).
|
|
4
|
+
*/
|
|
5
|
+
import { type SessionStart, type VSIPEvent } from "./events.js";
|
|
6
|
+
export type EventHandler = (event: VSIPEvent) => void;
|
|
7
|
+
export interface VSIPClientOptions {
|
|
8
|
+
apiKey: string;
|
|
9
|
+
url?: string;
|
|
10
|
+
audioFormat?: "int16" | "float32" | "mulaw8k";
|
|
11
|
+
mode?: "full" | "sidecar";
|
|
12
|
+
aec?: "client" | "server";
|
|
13
|
+
tickMs?: number;
|
|
14
|
+
sessionId?: string;
|
|
15
|
+
resumable?: boolean;
|
|
16
|
+
reconnect?: boolean;
|
|
17
|
+
maxReconnectAttempts?: number;
|
|
18
|
+
/** Inject a WebSocket implementation (e.g. `ws` on Node < 22). */
|
|
19
|
+
webSocketImpl?: typeof WebSocket;
|
|
20
|
+
}
|
|
21
|
+
export declare class VSIPError extends Error {
|
|
22
|
+
closeCode?: number | undefined;
|
|
23
|
+
constructor(message: string, closeCode?: number | undefined);
|
|
24
|
+
}
|
|
25
|
+
export declare class VSIPClient {
|
|
26
|
+
streamId: string | null;
|
|
27
|
+
session: SessionStart | null;
|
|
28
|
+
private ws;
|
|
29
|
+
private readonly opts;
|
|
30
|
+
private readonly handlers;
|
|
31
|
+
private closed;
|
|
32
|
+
private readonly WS;
|
|
33
|
+
constructor(options: VSIPClientOptions);
|
|
34
|
+
/** Register a handler for every event. Returns an unsubscribe fn. */
|
|
35
|
+
on(handler: EventHandler): () => void;
|
|
36
|
+
connect(): Promise<SessionStart>;
|
|
37
|
+
private buildUrl;
|
|
38
|
+
private dial;
|
|
39
|
+
private attachHandlers;
|
|
40
|
+
private onClose;
|
|
41
|
+
private emitLocal;
|
|
42
|
+
sendAudio(frame: ArrayBuffer | ArrayBufferView): void;
|
|
43
|
+
private sendCommand;
|
|
44
|
+
agentSpeaking(): void;
|
|
45
|
+
agentIdle(): void;
|
|
46
|
+
setBargeInMode(mode: string): void;
|
|
47
|
+
toggleLock(): void;
|
|
48
|
+
reset(): void;
|
|
49
|
+
close(): void;
|
|
50
|
+
}
|
package/dist/client.js
ADDED
|
@@ -0,0 +1,163 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* VSIPClient — WebSocket client for the VSIP Voice Events API.
|
|
3
|
+
* Browser (native WebSocket) and Node 18+ (global WebSocket / injected impl).
|
|
4
|
+
*/
|
|
5
|
+
import { parseEvent } from "./events.js";
|
|
6
|
+
const FATAL_CLOSE_CODES = new Set([4001, 4002, 4004, 4029]);
|
|
7
|
+
export class VSIPError extends Error {
|
|
8
|
+
constructor(message, closeCode) {
|
|
9
|
+
super(message);
|
|
10
|
+
this.closeCode = closeCode;
|
|
11
|
+
this.name = "VSIPError";
|
|
12
|
+
}
|
|
13
|
+
}
|
|
14
|
+
export class VSIPClient {
|
|
15
|
+
constructor(options) {
|
|
16
|
+
this.streamId = null;
|
|
17
|
+
this.session = null;
|
|
18
|
+
this.ws = null;
|
|
19
|
+
this.handlers = new Set();
|
|
20
|
+
this.closed = false;
|
|
21
|
+
this.opts = {
|
|
22
|
+
url: "wss://api.vsip.io/v1/stream",
|
|
23
|
+
audioFormat: "int16",
|
|
24
|
+
mode: "full",
|
|
25
|
+
aec: "client",
|
|
26
|
+
tickMs: 80,
|
|
27
|
+
resumable: true,
|
|
28
|
+
reconnect: true,
|
|
29
|
+
maxReconnectAttempts: 5,
|
|
30
|
+
...options,
|
|
31
|
+
};
|
|
32
|
+
const impl = options.webSocketImpl ?? globalThis.WebSocket;
|
|
33
|
+
if (!impl) {
|
|
34
|
+
throw new VSIPError("No WebSocket implementation found. In Node <22 pass webSocketImpl (e.g. from the 'ws' package).");
|
|
35
|
+
}
|
|
36
|
+
this.WS = impl;
|
|
37
|
+
}
|
|
38
|
+
/** Register a handler for every event. Returns an unsubscribe fn. */
|
|
39
|
+
on(handler) {
|
|
40
|
+
this.handlers.add(handler);
|
|
41
|
+
return () => this.handlers.delete(handler);
|
|
42
|
+
}
|
|
43
|
+
async connect() {
|
|
44
|
+
await this.dial(null);
|
|
45
|
+
return this.session;
|
|
46
|
+
}
|
|
47
|
+
buildUrl(resumeId) {
|
|
48
|
+
const p = new URLSearchParams({
|
|
49
|
+
audio_format: this.opts.audioFormat,
|
|
50
|
+
mode: this.opts.mode,
|
|
51
|
+
aec: this.opts.aec,
|
|
52
|
+
tick_ms: String(this.opts.tickMs),
|
|
53
|
+
resumable: this.opts.resumable ? "true" : "false",
|
|
54
|
+
});
|
|
55
|
+
if (this.opts.sessionId)
|
|
56
|
+
p.set("session_id", this.opts.sessionId);
|
|
57
|
+
if (resumeId)
|
|
58
|
+
p.set("resume", resumeId);
|
|
59
|
+
// Browsers can't set headers on the WS handshake — the token query param
|
|
60
|
+
// is the server's documented browser fallback.
|
|
61
|
+
p.set("token", this.opts.apiKey);
|
|
62
|
+
return `${this.opts.url}?${p.toString()}`;
|
|
63
|
+
}
|
|
64
|
+
dial(resumeId) {
|
|
65
|
+
return new Promise((resolve, reject) => {
|
|
66
|
+
const ws = new this.WS(this.buildUrl(resumeId));
|
|
67
|
+
ws.binaryType = "arraybuffer";
|
|
68
|
+
this.ws = ws;
|
|
69
|
+
const onFirst = (ev) => {
|
|
70
|
+
try {
|
|
71
|
+
const payload = JSON.parse(ev.data);
|
|
72
|
+
if (payload.type !== "session_start") {
|
|
73
|
+
reject(new VSIPError(`expected session_start, got ${payload.type}`));
|
|
74
|
+
return;
|
|
75
|
+
}
|
|
76
|
+
this.session = parseEvent(payload);
|
|
77
|
+
this.streamId = this.session.stream_id;
|
|
78
|
+
ws.removeEventListener("message", onFirst);
|
|
79
|
+
this.attachHandlers(ws);
|
|
80
|
+
resolve();
|
|
81
|
+
}
|
|
82
|
+
catch (e) {
|
|
83
|
+
reject(new VSIPError(`bad session_start: ${String(e)}`));
|
|
84
|
+
}
|
|
85
|
+
};
|
|
86
|
+
ws.addEventListener("message", onFirst);
|
|
87
|
+
ws.addEventListener("error", () => reject(new VSIPError("WebSocket connection failed")));
|
|
88
|
+
});
|
|
89
|
+
}
|
|
90
|
+
attachHandlers(ws) {
|
|
91
|
+
ws.addEventListener("message", (ev) => {
|
|
92
|
+
if (typeof ev.data !== "string")
|
|
93
|
+
return;
|
|
94
|
+
let payload;
|
|
95
|
+
try {
|
|
96
|
+
payload = JSON.parse(ev.data);
|
|
97
|
+
}
|
|
98
|
+
catch {
|
|
99
|
+
return;
|
|
100
|
+
}
|
|
101
|
+
const event = parseEvent(payload);
|
|
102
|
+
for (const h of this.handlers)
|
|
103
|
+
h(event);
|
|
104
|
+
});
|
|
105
|
+
ws.addEventListener("close", (ev) => this.onClose(ev));
|
|
106
|
+
}
|
|
107
|
+
async onClose(ev) {
|
|
108
|
+
if (this.closed)
|
|
109
|
+
return;
|
|
110
|
+
if (FATAL_CLOSE_CODES.has(ev.code)) {
|
|
111
|
+
this.emitLocal({ type: "error", message: `closed ${ev.code}`, raw: {} });
|
|
112
|
+
return;
|
|
113
|
+
}
|
|
114
|
+
if (!(this.opts.reconnect && this.opts.resumable && this.streamId))
|
|
115
|
+
return;
|
|
116
|
+
for (let attempt = 1; attempt <= this.opts.maxReconnectAttempts; attempt++) {
|
|
117
|
+
await new Promise((r) => setTimeout(r, Math.min(2 ** attempt, 10) * 1000));
|
|
118
|
+
if (this.closed)
|
|
119
|
+
return;
|
|
120
|
+
try {
|
|
121
|
+
await this.dial(this.streamId);
|
|
122
|
+
return;
|
|
123
|
+
}
|
|
124
|
+
catch {
|
|
125
|
+
/* retry */
|
|
126
|
+
}
|
|
127
|
+
}
|
|
128
|
+
}
|
|
129
|
+
emitLocal(event) {
|
|
130
|
+
for (const h of this.handlers)
|
|
131
|
+
h(event);
|
|
132
|
+
}
|
|
133
|
+
// ── Sending ──────────────────────────────────────────────────────────────
|
|
134
|
+
sendAudio(frame) {
|
|
135
|
+
if (this.ws && this.ws.readyState === this.WS.OPEN)
|
|
136
|
+
this.ws.send(frame);
|
|
137
|
+
}
|
|
138
|
+
sendCommand(command, fields = {}) {
|
|
139
|
+
if (this.ws && this.ws.readyState === this.WS.OPEN) {
|
|
140
|
+
this.ws.send(JSON.stringify({ command, ...fields }));
|
|
141
|
+
}
|
|
142
|
+
}
|
|
143
|
+
agentSpeaking() {
|
|
144
|
+
this.sendCommand("set_agent_state", { value: "speaking" });
|
|
145
|
+
}
|
|
146
|
+
agentIdle() {
|
|
147
|
+
this.sendCommand("set_agent_state", { value: "idle" });
|
|
148
|
+
}
|
|
149
|
+
setBargeInMode(mode) {
|
|
150
|
+
this.sendCommand("set_barge_in_mode", { mode });
|
|
151
|
+
}
|
|
152
|
+
toggleLock() {
|
|
153
|
+
this.sendCommand("toggle_lock");
|
|
154
|
+
}
|
|
155
|
+
reset() {
|
|
156
|
+
this.sendCommand("reset");
|
|
157
|
+
}
|
|
158
|
+
close() {
|
|
159
|
+
this.closed = true;
|
|
160
|
+
this.ws?.close();
|
|
161
|
+
this.ws = null;
|
|
162
|
+
}
|
|
163
|
+
}
|
package/dist/events.d.ts
ADDED
|
@@ -0,0 +1,93 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Typed VSIP events (schema_version 1.0). Parsing is tolerant: unknown types
|
|
3
|
+
* become `{ type, ...raw }` and unknown fields are preserved, so a pinned SDK
|
|
4
|
+
* keeps working as the additive-only protocol grows.
|
|
5
|
+
*/
|
|
6
|
+
export interface BaseEvent {
|
|
7
|
+
type: string;
|
|
8
|
+
timestamp?: number;
|
|
9
|
+
/** the complete wire payload, always present */
|
|
10
|
+
raw: Record<string, unknown>;
|
|
11
|
+
}
|
|
12
|
+
export interface SessionStart extends BaseEvent {
|
|
13
|
+
type: "session_start";
|
|
14
|
+
stream_id: string;
|
|
15
|
+
session_id: string;
|
|
16
|
+
plan_tier: string;
|
|
17
|
+
tick_ms: number;
|
|
18
|
+
mode: string;
|
|
19
|
+
resumed: boolean;
|
|
20
|
+
}
|
|
21
|
+
export interface SessionEnd extends BaseEvent {
|
|
22
|
+
type: "session_end";
|
|
23
|
+
stream_id: string;
|
|
24
|
+
duration_s: number;
|
|
25
|
+
billed_minutes: number;
|
|
26
|
+
}
|
|
27
|
+
export interface TurnStart extends BaseEvent {
|
|
28
|
+
type: "turn_start";
|
|
29
|
+
speaker_id: string;
|
|
30
|
+
audio_start: number;
|
|
31
|
+
interrupted_agent: boolean;
|
|
32
|
+
}
|
|
33
|
+
export interface TurnEnd extends BaseEvent {
|
|
34
|
+
type: "turn_end";
|
|
35
|
+
speaker_id: string;
|
|
36
|
+
audio_start: number;
|
|
37
|
+
duration_ms: number;
|
|
38
|
+
endpoint: string;
|
|
39
|
+
endpoint_prob: number | null;
|
|
40
|
+
interrupted_agent: boolean;
|
|
41
|
+
}
|
|
42
|
+
export interface BargeIn extends BaseEvent {
|
|
43
|
+
type: "barge_in";
|
|
44
|
+
speaker_id: string;
|
|
45
|
+
confidence: number;
|
|
46
|
+
vad: number;
|
|
47
|
+
}
|
|
48
|
+
export interface SpeakerVerification extends BaseEvent {
|
|
49
|
+
type: "speaker_verification";
|
|
50
|
+
speaker_id: string;
|
|
51
|
+
should_respond: boolean;
|
|
52
|
+
similarity: number | null;
|
|
53
|
+
audio_start: number;
|
|
54
|
+
interrupted_agent: boolean;
|
|
55
|
+
}
|
|
56
|
+
export interface SpeakerChange extends BaseEvent {
|
|
57
|
+
type: "speaker_change";
|
|
58
|
+
from: string;
|
|
59
|
+
to: string;
|
|
60
|
+
similarity: number;
|
|
61
|
+
}
|
|
62
|
+
export interface NoiseEvent extends BaseEvent {
|
|
63
|
+
type: "noise_event";
|
|
64
|
+
class: string;
|
|
65
|
+
confidence: number;
|
|
66
|
+
}
|
|
67
|
+
export interface LockStatus extends BaseEvent {
|
|
68
|
+
type: "lock_status";
|
|
69
|
+
status: string;
|
|
70
|
+
auto: boolean;
|
|
71
|
+
}
|
|
72
|
+
export interface Telemetry extends BaseEvent {
|
|
73
|
+
type: "telemetry";
|
|
74
|
+
vad: number;
|
|
75
|
+
speaker_id: string;
|
|
76
|
+
state: string;
|
|
77
|
+
noise_class: string;
|
|
78
|
+
}
|
|
79
|
+
export interface EnrollmentResult extends BaseEvent {
|
|
80
|
+
type: "enrollment_result";
|
|
81
|
+
success: boolean;
|
|
82
|
+
label: string;
|
|
83
|
+
}
|
|
84
|
+
export interface ErrorEvent extends BaseEvent {
|
|
85
|
+
type: "error";
|
|
86
|
+
message: string;
|
|
87
|
+
}
|
|
88
|
+
export type VSIPEvent = SessionStart | SessionEnd | TurnStart | TurnEnd | BargeIn | SpeakerVerification | SpeakerChange | NoiseEvent | LockStatus | Telemetry | EnrollmentResult | ErrorEvent | (BaseEvent & {
|
|
89
|
+
type: string;
|
|
90
|
+
});
|
|
91
|
+
export declare function parseEvent(payload: Record<string, unknown>): VSIPEvent;
|
|
92
|
+
/** Helper: is a lock_status locked, and to whom? */
|
|
93
|
+
export declare function lockedSpeaker(e: LockStatus): string | null;
|
package/dist/events.js
ADDED
|
@@ -0,0 +1,14 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Typed VSIP events (schema_version 1.0). Parsing is tolerant: unknown types
|
|
3
|
+
* become `{ type, ...raw }` and unknown fields are preserved, so a pinned SDK
|
|
4
|
+
* keeps working as the additive-only protocol grows.
|
|
5
|
+
*/
|
|
6
|
+
export function parseEvent(payload) {
|
|
7
|
+
return { ...payload, type: String(payload.type ?? "unknown"), raw: payload };
|
|
8
|
+
}
|
|
9
|
+
/** Helper: is a lock_status locked, and to whom? */
|
|
10
|
+
export function lockedSpeaker(e) {
|
|
11
|
+
return e.status.startsWith("locked") && e.status.includes(":")
|
|
12
|
+
? e.status.split(":", 2)[1]
|
|
13
|
+
: null;
|
|
14
|
+
}
|
package/dist/index.d.ts
ADDED
|
@@ -0,0 +1,6 @@
|
|
|
1
|
+
export { VSIPClient, VSIPError } from "./client.js";
|
|
2
|
+
export type { VSIPClientOptions, EventHandler } from "./client.js";
|
|
3
|
+
export { VoiceSession } from "./session.js";
|
|
4
|
+
export type { Turn, VoiceSessionOptions } from "./session.js";
|
|
5
|
+
export { parseEvent, lockedSpeaker } from "./events.js";
|
|
6
|
+
export type { VSIPEvent, BaseEvent, SessionStart, SessionEnd, TurnStart, TurnEnd, BargeIn, SpeakerVerification, SpeakerChange, NoiseEvent, LockStatus, Telemetry, EnrollmentResult, ErrorEvent, } from "./events.js";
|
package/dist/index.js
ADDED
|
@@ -0,0 +1,57 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* VoiceSession — the recommended integration layer for the JS SDK. Mirrors the
|
|
3
|
+
* Python VoiceSession: onset-lookback turn slicing, pre-lock fast path +
|
|
4
|
+
* post-lock speaker_verification gating, queued turn dispatch, capped idle
|
|
5
|
+
* buffer. Audio is int16 PCM (the browser default).
|
|
6
|
+
*/
|
|
7
|
+
import { VSIPClient } from "./client.js";
|
|
8
|
+
import type { BargeIn, VSIPEvent } from "./events.js";
|
|
9
|
+
export interface Turn {
|
|
10
|
+
/** int16 PCM samples for the turn, including onset lookback */
|
|
11
|
+
audio: Int16Array;
|
|
12
|
+
sampleRate: number;
|
|
13
|
+
speakerId: string;
|
|
14
|
+
shouldRespond: boolean;
|
|
15
|
+
verified: boolean;
|
|
16
|
+
similarity: number | null;
|
|
17
|
+
interruptedAgent: boolean;
|
|
18
|
+
durationMs: number;
|
|
19
|
+
endpoint: string;
|
|
20
|
+
}
|
|
21
|
+
export interface VoiceSessionOptions {
|
|
22
|
+
onTurn?: (turn: Turn) => void | Promise<void>;
|
|
23
|
+
onBargeIn?: (e: BargeIn) => void;
|
|
24
|
+
onEvent?: (e: VSIPEvent) => void;
|
|
25
|
+
verifyTimeoutMs?: number;
|
|
26
|
+
lookbackS?: number;
|
|
27
|
+
bargeInLookbackS?: number;
|
|
28
|
+
idleBufferCapS?: number;
|
|
29
|
+
postTurnKeepS?: number;
|
|
30
|
+
autoAgentIdleOnBargeIn?: boolean;
|
|
31
|
+
sampleRate?: number;
|
|
32
|
+
}
|
|
33
|
+
export declare class VoiceSession {
|
|
34
|
+
private readonly client;
|
|
35
|
+
private readonly o;
|
|
36
|
+
private chunks;
|
|
37
|
+
private totalSamples;
|
|
38
|
+
private inTurn;
|
|
39
|
+
private turnSliceStart;
|
|
40
|
+
private locked;
|
|
41
|
+
private pending;
|
|
42
|
+
private queue;
|
|
43
|
+
private unsub;
|
|
44
|
+
constructor(client: VSIPClient, options?: VoiceSessionOptions);
|
|
45
|
+
start(): void;
|
|
46
|
+
stop(): void;
|
|
47
|
+
/** Feed one int16 PCM frame: buffered locally for turn slicing AND streamed. */
|
|
48
|
+
sendAudio(frame: Int16Array): void;
|
|
49
|
+
/** Resolves once no user turn is open (don't talk over the caller). */
|
|
50
|
+
speakGate(pollMs?: number): Promise<void>;
|
|
51
|
+
private absStart;
|
|
52
|
+
private sliceFrom;
|
|
53
|
+
private trimToLast;
|
|
54
|
+
private handle;
|
|
55
|
+
private dispatch;
|
|
56
|
+
private dispatchRaw;
|
|
57
|
+
}
|
package/dist/session.js
ADDED
|
@@ -0,0 +1,179 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* VoiceSession — the recommended integration layer for the JS SDK. Mirrors the
|
|
3
|
+
* Python VoiceSession: onset-lookback turn slicing, pre-lock fast path +
|
|
4
|
+
* post-lock speaker_verification gating, queued turn dispatch, capped idle
|
|
5
|
+
* buffer. Audio is int16 PCM (the browser default).
|
|
6
|
+
*/
|
|
7
|
+
const VERIFY_MATCH_TOLERANCE_S = 1.5;
|
|
8
|
+
export class VoiceSession {
|
|
9
|
+
constructor(client, options = {}) {
|
|
10
|
+
this.chunks = [];
|
|
11
|
+
this.totalSamples = 0;
|
|
12
|
+
this.inTurn = false;
|
|
13
|
+
this.turnSliceStart = 0;
|
|
14
|
+
this.locked = false;
|
|
15
|
+
this.pending = null;
|
|
16
|
+
this.queue = Promise.resolve();
|
|
17
|
+
this.unsub = null;
|
|
18
|
+
this.client = client;
|
|
19
|
+
this.o = {
|
|
20
|
+
onTurn: options.onTurn ?? (() => { }),
|
|
21
|
+
onBargeIn: options.onBargeIn ?? (() => { }),
|
|
22
|
+
onEvent: options.onEvent ?? (() => { }),
|
|
23
|
+
verifyTimeoutMs: options.verifyTimeoutMs ?? 12000,
|
|
24
|
+
lookbackS: options.lookbackS ?? 1.0,
|
|
25
|
+
bargeInLookbackS: options.bargeInLookbackS ?? 1.5,
|
|
26
|
+
idleBufferCapS: options.idleBufferCapS ?? 35,
|
|
27
|
+
postTurnKeepS: options.postTurnKeepS ?? 2,
|
|
28
|
+
autoAgentIdleOnBargeIn: options.autoAgentIdleOnBargeIn ?? true,
|
|
29
|
+
sampleRate: options.sampleRate ?? 16000,
|
|
30
|
+
};
|
|
31
|
+
}
|
|
32
|
+
start() {
|
|
33
|
+
this.unsub = this.client.on((e) => this.handle(e));
|
|
34
|
+
}
|
|
35
|
+
stop() {
|
|
36
|
+
this.unsub?.();
|
|
37
|
+
this.unsub = null;
|
|
38
|
+
if (this.pending) {
|
|
39
|
+
clearTimeout(this.pending.timer);
|
|
40
|
+
this.pending = null;
|
|
41
|
+
}
|
|
42
|
+
}
|
|
43
|
+
/** Feed one int16 PCM frame: buffered locally for turn slicing AND streamed. */
|
|
44
|
+
sendAudio(frame) {
|
|
45
|
+
this.chunks.push(frame);
|
|
46
|
+
this.totalSamples += frame.length;
|
|
47
|
+
if (!this.inTurn) {
|
|
48
|
+
const cap = this.o.idleBufferCapS * this.o.sampleRate;
|
|
49
|
+
while (this.totalSamples - (this.chunks[0]?.length ?? 0) > cap && this.chunks.length > 1) {
|
|
50
|
+
this.totalSamples -= this.chunks.shift().length;
|
|
51
|
+
}
|
|
52
|
+
}
|
|
53
|
+
// Pass the view directly — WebSocket.send transmits exactly the view's
|
|
54
|
+
// bytes (avoids ArrayBuffer/SharedArrayBuffer copies and typing friction).
|
|
55
|
+
this.client.sendAudio(frame);
|
|
56
|
+
}
|
|
57
|
+
/** Resolves once no user turn is open (don't talk over the caller). */
|
|
58
|
+
async speakGate(pollMs = 100) {
|
|
59
|
+
while (this.inTurn)
|
|
60
|
+
await new Promise((r) => setTimeout(r, pollMs));
|
|
61
|
+
}
|
|
62
|
+
// ── internals ──────────────────────────────────────────────────────────
|
|
63
|
+
absStart() {
|
|
64
|
+
// absolute sample index of the oldest buffered sample
|
|
65
|
+
return this.totalSamples - this.chunks.reduce((n, c) => n + c.length, 0);
|
|
66
|
+
}
|
|
67
|
+
sliceFrom(startSample) {
|
|
68
|
+
let cursor = this.absStart();
|
|
69
|
+
const parts = [];
|
|
70
|
+
for (const c of this.chunks) {
|
|
71
|
+
const end = cursor + c.length;
|
|
72
|
+
if (end > startSample) {
|
|
73
|
+
const off = Math.max(0, startSample - cursor);
|
|
74
|
+
parts.push(c.subarray(off));
|
|
75
|
+
}
|
|
76
|
+
cursor = end;
|
|
77
|
+
}
|
|
78
|
+
const total = parts.reduce((n, p) => n + p.length, 0);
|
|
79
|
+
const out = new Int16Array(total);
|
|
80
|
+
let i = 0;
|
|
81
|
+
for (const p of parts) {
|
|
82
|
+
out.set(p, i);
|
|
83
|
+
i += p.length;
|
|
84
|
+
}
|
|
85
|
+
return out;
|
|
86
|
+
}
|
|
87
|
+
trimToLast(seconds) {
|
|
88
|
+
const keep = seconds * this.o.sampleRate;
|
|
89
|
+
while (this.totalSamples - (this.absStart() + (this.chunks[0]?.length ?? 0)) >= keep && this.chunks.length > 1) {
|
|
90
|
+
this.chunks.shift();
|
|
91
|
+
}
|
|
92
|
+
}
|
|
93
|
+
handle(e) {
|
|
94
|
+
this.o.onEvent(e);
|
|
95
|
+
switch (e.type) {
|
|
96
|
+
case "turn_start": {
|
|
97
|
+
const ts = e;
|
|
98
|
+
this.inTurn = true;
|
|
99
|
+
const lb = ts.interrupted_agent ? this.o.bargeInLookbackS : this.o.lookbackS;
|
|
100
|
+
this.turnSliceStart = Math.max(0, this.totalSamples - lb * this.o.sampleRate);
|
|
101
|
+
break;
|
|
102
|
+
}
|
|
103
|
+
case "turn_end": {
|
|
104
|
+
if (!this.inTurn)
|
|
105
|
+
break;
|
|
106
|
+
this.inTurn = false;
|
|
107
|
+
const te = e;
|
|
108
|
+
const audio = this.sliceFrom(this.turnSliceStart);
|
|
109
|
+
this.trimToLast(this.o.postTurnKeepS);
|
|
110
|
+
if (!this.locked) {
|
|
111
|
+
this.dispatch(te, audio, true, false, null);
|
|
112
|
+
}
|
|
113
|
+
else {
|
|
114
|
+
if (this.pending)
|
|
115
|
+
clearTimeout(this.pending.timer);
|
|
116
|
+
const timer = setTimeout(() => {
|
|
117
|
+
if (this.pending && this.pending.timer === timer) {
|
|
118
|
+
const p = this.pending;
|
|
119
|
+
this.pending = null;
|
|
120
|
+
this.dispatch(p.event, p.audio, false, false, null);
|
|
121
|
+
}
|
|
122
|
+
}, this.o.verifyTimeoutMs);
|
|
123
|
+
this.pending = { audioStart: te.audio_start, audio, event: te, timer };
|
|
124
|
+
}
|
|
125
|
+
break;
|
|
126
|
+
}
|
|
127
|
+
case "speaker_verification": {
|
|
128
|
+
const v = e;
|
|
129
|
+
const p = this.pending;
|
|
130
|
+
if (!p)
|
|
131
|
+
break;
|
|
132
|
+
if (v.audio_start && p.audioStart && Math.abs(v.audio_start - p.audioStart) > VERIFY_MATCH_TOLERANCE_S)
|
|
133
|
+
break;
|
|
134
|
+
clearTimeout(p.timer);
|
|
135
|
+
this.pending = null;
|
|
136
|
+
this.dispatchRaw({
|
|
137
|
+
speakerId: v.speaker_id,
|
|
138
|
+
durationMs: p.event.duration_ms,
|
|
139
|
+
endpoint: p.event.endpoint,
|
|
140
|
+
interruptedAgent: v.interrupted_agent,
|
|
141
|
+
}, p.audio, v.should_respond, true, v.similarity);
|
|
142
|
+
break;
|
|
143
|
+
}
|
|
144
|
+
case "barge_in": {
|
|
145
|
+
if (this.o.autoAgentIdleOnBargeIn)
|
|
146
|
+
this.client.agentIdle();
|
|
147
|
+
this.o.onBargeIn(e);
|
|
148
|
+
break;
|
|
149
|
+
}
|
|
150
|
+
case "lock_status": {
|
|
151
|
+
this.locked = e.status.startsWith("locked");
|
|
152
|
+
break;
|
|
153
|
+
}
|
|
154
|
+
}
|
|
155
|
+
}
|
|
156
|
+
dispatch(te, audio, shouldRespond, verified, similarity) {
|
|
157
|
+
this.dispatchRaw({
|
|
158
|
+
speakerId: te.speaker_id,
|
|
159
|
+
durationMs: te.duration_ms,
|
|
160
|
+
endpoint: te.endpoint,
|
|
161
|
+
interruptedAgent: te.interrupted_agent,
|
|
162
|
+
}, audio, shouldRespond, verified, similarity);
|
|
163
|
+
}
|
|
164
|
+
dispatchRaw(meta, audio, shouldRespond, verified, similarity) {
|
|
165
|
+
const turn = {
|
|
166
|
+
audio,
|
|
167
|
+
sampleRate: this.o.sampleRate,
|
|
168
|
+
speakerId: meta.speakerId,
|
|
169
|
+
shouldRespond,
|
|
170
|
+
verified,
|
|
171
|
+
similarity,
|
|
172
|
+
interruptedAgent: meta.interruptedAgent,
|
|
173
|
+
durationMs: meta.durationMs,
|
|
174
|
+
endpoint: meta.endpoint,
|
|
175
|
+
};
|
|
176
|
+
// Queue so async handlers run in order and never drop a turn.
|
|
177
|
+
this.queue = this.queue.then(() => this.o.onTurn(turn)).catch(() => { });
|
|
178
|
+
}
|
|
179
|
+
}
|
package/package.json
ADDED
|
@@ -0,0 +1,31 @@
|
|
|
1
|
+
{
|
|
2
|
+
"name": "vsip-sdk",
|
|
3
|
+
"version": "0.1.0",
|
|
4
|
+
"description": "Official JavaScript/TypeScript SDK for the VSIP Voice Events API — real-time turn detection, identity-gated barge-in, and speaker verification for voice agents.",
|
|
5
|
+
"type": "module",
|
|
6
|
+
"main": "./dist/index.js",
|
|
7
|
+
"types": "./dist/index.d.ts",
|
|
8
|
+
"exports": {
|
|
9
|
+
".": {
|
|
10
|
+
"types": "./dist/index.d.ts",
|
|
11
|
+
"import": "./dist/index.js"
|
|
12
|
+
}
|
|
13
|
+
},
|
|
14
|
+
"files": ["dist", "README.md", "LICENSE"],
|
|
15
|
+
"sideEffects": false,
|
|
16
|
+
"scripts": {
|
|
17
|
+
"build": "tsc -p tsconfig.json",
|
|
18
|
+
"typecheck": "tsc -p tsconfig.json --noEmit",
|
|
19
|
+
"test": "vitest run"
|
|
20
|
+
},
|
|
21
|
+
"keywords": [
|
|
22
|
+
"voice", "voice-agent", "barge-in", "speaker-verification",
|
|
23
|
+
"turn-detection", "vad", "websocket", "realtime"
|
|
24
|
+
],
|
|
25
|
+
"license": "MIT",
|
|
26
|
+
"engines": { "node": ">=18" },
|
|
27
|
+
"devDependencies": {
|
|
28
|
+
"typescript": "^5.4.0",
|
|
29
|
+
"vitest": "^1.6.0"
|
|
30
|
+
}
|
|
31
|
+
}
|