ulai-js-sdk 0.1.0 → 0.1.2

package/dist/events.js

@@ -8,63 +8,170 @@ const eventemitter3_1 = __importDefault(require("eventemitter3"));
 class EventsClient extends eventemitter3_1.default {
     ws = null;
     url;
+    sessionId = null;
+    eventsToken = null;
+    lastEventId = null;
+    isClosedManually = false;
+    retryAttempts = 0;
+    maxRetries = 5;
+    reconnectTimer = null;
+    heartbeatTimer = null;
     constructor(cpBase) {
         super();
-        const scheme = cpBase.startsWith('https') ? 'wss' : 'ws';
-        const host = cpBase.replace(/^https?:\/\//, '').replace(/\/$/, '');
-        this.url = `${scheme}://${host}/api/v1/events`;
+        this.url = this.buildUrl(cpBase);
+    }
+    buildUrl(cpBase) {
+        const cleanHost = cpBase
+            .replace(/^https?:\/\//, '')
+            .replace(/^wss?:\/\//, '')
+            .replace(/\/$/, '');
+        const isLocal = cleanHost.startsWith('localhost') ||
+            cleanHost.startsWith('127.0.0.1');
+        let scheme = 'wss';
+        if (isLocal) {
+            scheme = 'ws';
+        }
+        else if (typeof window !== 'undefined' && window.location.protocol === 'http:') {
+            scheme = 'ws';
+        }
+        else if (cpBase.startsWith('http://')) {
+            scheme = 'ws';
+        }
+        return `${scheme}://${cleanHost}/api/v1/events`;
     }
     connect(sessionId, eventsToken) {
-        this.ws = new WebSocket(this.url);
-        this.ws.onopen = () => {
-            this.ws?.send(JSON.stringify({
-                action: 'subscribe',
-                scope: 'session',
-                session_id: sessionId,
-                token: eventsToken
-            }));
-        };
-        this.ws.onmessage = (event) => {
-            try {
-                const msg = JSON.parse(event.data);
-                if (msg.type === 'subscribed') {
-                    this.emit('connected', msg);
-                }
-                else if (msg.type === 'transcript_delta') {
-                    this.emit('transcript_delta', msg);
+        this.sessionId = sessionId;
+        this.eventsToken = eventsToken;
+        this.isClosedManually = false;
+        this.cleanupSocket();
+        try {
+            this.ws = new WebSocket(this.url);
+            this.ws.onopen = () => {
+                this.retryAttempts = 0;
+                this.sendSubscribe();
+                this.startHeartbeat();
+            };
+            this.ws.onmessage = (event) => {
+                try {
+                    const msg = JSON.parse(event.data);
+                    if (msg.id) {
+                        this.lastEventId = msg.id;
+                    }
+                    if (msg.type === 'subscribed') {
+                        this.emit('connected', msg);
+                    }
+                    else if (msg.type === 'transcript_delta') {
+                        this.emit('transcript_delta', msg);
+                    }
+                    else if (msg.type === 'roster') {
+                        this.emit('roster', msg);
+                    }
+                    else if (msg.type === 'session_terminated') {
+                        this.emit('session_terminated', msg);
+                    }
+                    else if (msg.type === 'participant_joined') {
+                        this.emit('participant_joined', msg);
+                    }
+                    else if (msg.type === 'participant_left') {
+                        this.emit('participant_left', msg);
+                    }
+                    else if (msg.type === 'error') {
+                        this.emit('error', msg);
+                    }
                 }
-                else if (msg.type === 'roster') {
-                    this.emit('roster', msg);
+                catch (err) {
+                    console.error('[Events] Failed to parse message', err);
                 }
-                else if (msg.type === 'session_terminated') {
-                    this.emit('session_terminated', msg);
+            };
+            this.ws.onclose = (event) => {
+                this.stopHeartbeat();
+                if (this.isClosedManually) {
+                    this.emit('disconnected');
+                    return;
                 }
-                else if (msg.type === 'participant_joined') {
-                    this.emit('participant_joined', msg);
+                if (this.retryAttempts < this.maxRetries) {
+                    this.retryAttempts++;
+                    const delay = Math.min(1000 * Math.pow(1.5, this.retryAttempts), 10000);
+                    console.warn(`[Events] Event socket closed. Reconnecting in ${Math.round(delay)}ms (attempt ${this.retryAttempts}/${this.maxRetries})...`);
+                    this.emit('reconnecting', { attempt: this.retryAttempts });
+                    this.reconnectTimer = setTimeout(() => {
+                        if (!this.isClosedManually && this.sessionId && this.eventsToken) {
+                            this.connect(this.sessionId, this.eventsToken);
+                        }
+                    }, delay);
                 }
-                else if (msg.type === 'participant_left') {
-                    this.emit('participant_left', msg);
+                else {
+                    console.error('[Events] Max reconnect attempts exceeded for events socket');
+                    this.emit('disconnected', event);
                 }
-                else if (msg.type === 'error') {
-                    this.emit('error', msg);
+            };
+            this.ws.onerror = (error) => {
+                this.emit('error', error);
+            };
+        }
+        catch (err) {
+            this.emit('error', err);
+        }
+    }
+    sendSubscribe() {
+        if (!this.ws || this.ws.readyState !== WebSocket.OPEN || !this.sessionId || !this.eventsToken) {
+            return;
+        }
+        const payload = {
+            action: 'subscribe',
+            scope: 'session',
+            session_id: this.sessionId,
+            token: this.eventsToken,
+        };
+        if (this.lastEventId) {
+            payload.last_event_id = this.lastEventId;
+        }
+        try {
+            this.ws.send(JSON.stringify(payload));
+        }
+        catch (err) {
+            console.error('[Events] Failed to send subscription', err);
+        }
+    }
+    startHeartbeat() {
+        this.stopHeartbeat();
+        this.heartbeatTimer = setInterval(() => {
+            if (this.ws && this.ws.readyState === WebSocket.OPEN) {
+                try {
+                    this.ws.send(JSON.stringify({ action: 'ping' }));
                 }
+                catch (_) { }
             }
-            catch (err) {
-                console.error('Failed to parse events message', err);
-            }
-        };
-        this.ws.onclose = () => {
-            this.emit('disconnected');
-        };
-        this.ws.onerror = (error) => {
-            this.emit('error', error);
-        };
+        }, 25000);
     }
-    close() {
+    stopHeartbeat() {
+        if (this.heartbeatTimer) {
+            clearInterval(this.heartbeatTimer);
+            this.heartbeatTimer = null;
+        }
+    }
+    cleanupSocket() {
+        this.stopHeartbeat();
+        if (this.reconnectTimer) {
+            clearTimeout(this.reconnectTimer);
+            this.reconnectTimer = null;
+        }
         if (this.ws) {
-            this.ws.close();
+            this.ws.onopen = null;
+            this.ws.onmessage = null;
+            this.ws.onclose = null;
+            this.ws.onerror = null;
+            try {
+                this.ws.close();
+            }
+            catch (_) { }
             this.ws = null;
         }
     }
+    close() {
+        this.isClosedManually = true;
+        this.cleanupSocket();
+        this.emit('disconnected');
+    }
 }
 exports.EventsClient = EventsClient;

package/dist/room.d.ts

@@ -5,14 +5,42 @@ export declare class Room extends EventEmitter<RoomEvent> {
     private signaling;
     private eventsClient;
     private peerConnection;
+    private audioTransceiver;
     private localStream;
     private remoteStreams;
     roomId: string | null;
     participantId: string | null;
+    private readonly isPolite;
+    private makingOffer;
+    private candidateQueue;
+    private signalQueue;
     constructor(cpBase: string);
     connect(options: JoinRoomOptions): Promise<void>;
     private setupEventsClient;
     private setupSignaling;
-    publishAudio(): Promise<void>;
+    private handleSignal;
+    private safeNegotiate;
+    private handleIncomingCandidate;
+    private drainCandidateQueue;
+    /**
+     * Publishes local microphone audio to the room.
+     */
+    publishAudio(): Promise<MediaStream>;
+    /**
+     * Sets microphone mute state.
+     */
+    setMuted(muted: boolean): void;
+    /**
+     * Returns whether the microphone is currently muted.
+     */
+    isMuted(): boolean;
+    /**
+     * Gets the active local media stream.
+     */
+    getLocalStream(): MediaStream | null;
+    /**
+     * Gets all subscribed remote audio streams.
+     */
+    getRemoteStreams(): MediaStream[];
     disconnect(): void;
 }

package/dist/room.js

@@ -12,10 +12,18 @@ class Room extends eventemitter3_1.default {
     signaling = null;
     eventsClient = null;
     peerConnection = null;
+    audioTransceiver = null;
     localStream = null;
     remoteStreams = new Map();
     roomId = null;
     participantId = null;
+    // Perfect Negotiation State (Client is the polite peer)
+    isPolite = true;
+    makingOffer = false;
+    candidateQueue = [];
+    // Signaling messages are handled one at a time, in arrival order. Handled concurrently, a bridge
+    // offer could be processed in the middle of our own offer and dropped.
+    signalQueue = Promise.resolve();
     constructor(cpBase) {
         super();
         this.cpBase = cpBase;
@@ -23,43 +31,75 @@ class Room extends eventemitter3_1.default {
     async connect(options) {
         this.roomId = options.roomId;
         this.participantId = options.participantId || '';
-        // 1. Setup Control Plane Events
+        // 1. Setup Control Plane Events (Transcripts, Roster, Session Lifecycle)
         if (options.eventsToken) {
             this.eventsClient = new events_1.EventsClient(this.cpBase);
             this.setupEventsClient();
             this.eventsClient.connect(this.roomId, options.eventsToken);
         }
-        // 2. Setup WebRTC Signaling if nodeAddress and ticket are provided
+        // 2. Setup WebRTC Signaling & Media
         if (options.nodeAddress && options.ticket) {
             this.signaling = new signaling_1.SignalingClient(options.nodeAddress, options.ticket);
             this.setupSignaling();
-            this.peerConnection = new RTCPeerConnection({
-                iceServers: [
-                    { urls: 'stun:stun.l.google.com:19302' },
-                    { urls: ['turn:turn.ulai.in:3478', 'turns:turn.ulai.in:5349'], username: 'ulai', credential: 'xulaizrt' }
-                ]
-            });
+            const defaultIceServers = [
+                { urls: 'stun:stun.l.google.com:19302' },
+            ];
+            const iceServers = options.iceServers && options.iceServers.length > 0
+                ? options.iceServers
+                : defaultIceServers;
+            this.peerConnection = new RTCPeerConnection({ iceServers });
+            // Every local change that needs an offer (the initial transceiver, publishing the mic, an
+            // offer rolled back by a collision) fires this. Offers sent before signaling opens wait in
+            // its outbox.
+            this.peerConnection.onnegotiationneeded = () => {
+                void this.safeNegotiate();
+            };
+            // Pre-allocate audio transceiver as recvonly so the initial offer immediately
+            // negotiates audio codec capabilities (m=audio) and is ready for agent downlink.
+            try {
+                this.audioTransceiver = this.peerConnection.addTransceiver('audio', {
+                    direction: 'recvonly',
+                });
+            }
+            catch (err) {
+                console.warn('[Room] Error adding initial audio transceiver:', err);
+            }
             this.peerConnection.onicecandidate = (event) => {
                 if (event.candidate && this.signaling) {
-                    this.signaling.send({ type: 'candidate', candidate: JSON.stringify(event.candidate.toJSON()) });
+                    this.signaling.send({
+                        type: 'candidate',
+                        candidate: JSON.stringify(event.candidate.toJSON()),
+                    });
                 }
             };
+            // Resilient ontrack handler: works with stream-grouped tracks and bare Pion SFU tracks
             this.peerConnection.ontrack = (event) => {
-                const stream = event.streams[0];
-                if (stream) {
-                    this.remoteStreams.set(stream.id, stream);
-                    this.emit('trackSubscribed', event.track, stream);
-                }
+                const stream = event.streams && event.streams[0]
+                    ? event.streams[0]
+                    : new MediaStream([event.track]);
+                this.remoteStreams.set(stream.id, stream);
+                this.emit('trackSubscribed', event.track, stream);
+                event.track.onended = () => {
+                    this.remoteStreams.delete(stream.id);
+                    this.emit('trackUnsubscribed', event.track, stream);
+                };
             };
             this.peerConnection.onconnectionstatechange = () => {
-                switch (this.peerConnection?.connectionState) {
+                const state = this.peerConnection?.connectionState;
+                switch (state) {
                     case 'connected':
                         this.emit('connected');
                         break;
+                    case 'connecting':
+                        this.emit('reconnecting');
+                        break;
                     case 'disconnected':
                     case 'failed':
                         this.emit('disconnected');
                         break;
+                    case 'closed':
+                        this.emit('disconnected');
+                        break;
                 }
             };
             this.signaling.connect();
@@ -81,65 +121,212 @@ class Room extends eventemitter3_1.default {
     setupSignaling() {
         if (!this.signaling)
             return;
-        this.signaling.on('connected', async () => {
-            // Create initial offer when signaling connects
-            if (this.peerConnection) {
-                const offer = await this.peerConnection.createOffer();
-                await this.peerConnection.setLocalDescription(offer);
-                this.signaling.send({ type: 'offer', sdp: this.peerConnection.localDescription?.sdp });
-            }
+        // The initial offer comes from onnegotiationneeded and is flushed from the outbox on connect.
+        this.signaling.on('reconnected', async () => {
+            this.emit('reconnected');
+            await this.safeNegotiate();
         });
-        this.signaling.on('message', async (msg) => {
-            if (!this.peerConnection)
-                return;
-            try {
-                if (msg.type === 'offer' || msg.type === 'restart-offer') {
-                    await this.peerConnection.setRemoteDescription(new RTCSessionDescription({ type: 'offer', sdp: msg.sdp }));
-                    const answer = await this.peerConnection.createAnswer();
-                    await this.peerConnection.setLocalDescription(answer);
-                    this.signaling.send({ type: 'answer', sdp: this.peerConnection.localDescription?.sdp });
+        this.signaling.on('message', (msg) => {
+            this.signalQueue = this.signalQueue.then(() => this.handleSignal(msg));
+        });
+        this.signaling.on('disconnected', () => {
+            this.emit('disconnected');
+        });
+    }
+    async handleSignal(msg) {
+        const pc = this.peerConnection;
+        if (!pc)
+            return;
+        try {
+            if (msg.type === 'offer' || msg.type === 'restart-offer') {
+                if (!msg.sdp)
+                    return;
+                // Perfect negotiation: detect collision (SDP Glare)
+                const offerCollision = this.makingOffer || pc.signalingState !== 'stable';
+                if (offerCollision && !this.isPolite) {
+                    console.warn('[Room] Impolite collision: ignoring incoming offer');
+                    return;
                 }
-                else if (msg.type === 'answer') {
-                    await this.peerConnection.setRemoteDescription(new RTCSessionDescription({ type: 'answer', sdp: msg.sdp }));
+                if (offerCollision) {
+                    console.log('[Room] Polite collision: rolling back local offer to accept server offer');
                 }
-                else if (msg.type === 'candidate' && msg.candidate) {
-                    await this.peerConnection.addIceCandidate(JSON.parse(msg.candidate));
+                // On a collision this implicitly rolls back our own offer, even one still being made. An
+                // explicit rollback throws if we're stable by the time it runs, which is what happens when
+                // the bridge sends its answer and then its own offer right away (someone already talking),
+                // and the bridge's offer was dropped. Whatever our offer carried is offered again via
+                // negotiationneeded once we're back to stable.
+                await pc.setRemoteDescription({ type: 'offer', sdp: msg.sdp });
+                await this.drainCandidateQueue();
+                await pc.setLocalDescription();
+                this.signaling?.send({
+                    type: 'answer',
+                    sdp: pc.localDescription?.sdp,
+                });
+            }
+            else if (msg.type === 'answer') {
+                if (!msg.sdp)
+                    return;
+                // An answer to an offer we've since rolled back has nothing to apply to.
+                if (pc.signalingState !== 'have-local-offer') {
+                    console.warn('[Room] Ignoring answer in signalingState', pc.signalingState);
+                    return;
                 }
+                await pc.setRemoteDescription({ type: 'answer', sdp: msg.sdp });
+                await this.drainCandidateQueue();
             }
-            catch (err) {
-                console.error('Error handling signaling message', err);
+            else if (msg.type === 'candidate' && msg.candidate) {
+                const candidateInit = JSON.parse(msg.candidate);
+                await this.handleIncomingCandidate(candidateInit);
             }
-        });
-        this.signaling.on('disconnected', () => {
-            this.emit('disconnected');
-        });
+        }
+        catch (err) {
+            console.error('[Room] Error handling signaling message:', err);
+            this.emit('error', err);
+        }
     }
+    async safeNegotiate() {
+        if (!this.peerConnection || this.makingOffer)
+            return;
+        // Mid-negotiation, setLocalDescription() would make an answer, not an offer. negotiationneeded
+        // fires again once the current exchange completes.
+        if (this.peerConnection.signalingState !== 'stable')
+            return;
+        try {
+            this.makingOffer = true;
+            // Without an argument the offer is created when it's applied, so it can't go stale against
+            // an offer from the bridge that arrives meanwhile.
+            await this.peerConnection.setLocalDescription();
+            if (this.peerConnection.pendingLocalDescription?.type !== 'offer')
+                return; // rolled back
+            this.signaling?.send({
+                type: 'offer',
+                sdp: this.peerConnection.localDescription?.sdp,
+            });
+        }
+        catch (err) {
+            console.error('[Room] safeNegotiate error:', err);
+        }
+        finally {
+            this.makingOffer = false;
+        }
+    }
+    async handleIncomingCandidate(candidate) {
+        if (!this.peerConnection?.remoteDescription || !this.peerConnection.remoteDescription.type) {
+            // Buffer candidates if remote description is not set yet
+            this.candidateQueue.push(candidate);
+            return;
+        }
+        try {
+            await this.peerConnection.addIceCandidate(candidate);
+        }
+        catch (err) {
+            console.warn('[Room] Failed to add ICE candidate:', err);
+        }
+    }
+    async drainCandidateQueue() {
+        while (this.candidateQueue.length > 0) {
+            const candidate = this.candidateQueue.shift();
+            if (candidate && this.peerConnection?.remoteDescription) {
+                try {
+                    await this.peerConnection.addIceCandidate(candidate);
+                }
+                catch (err) {
+                    console.warn('[Room] Error draining candidate:', err);
+                }
+            }
+        }
+    }
+    /**
+     * Publishes local microphone audio to the room.
+     */
     async publishAudio() {
         try {
-            this.localStream = await navigator.mediaDevices.getUserMedia({ audio: true });
-            if (this.peerConnection && this.localStream) {
-                this.localStream.getTracks().forEach((track) => {
-                    this.peerConnection.addTrack(track, this.localStream);
-                });
-                // Renegotiate
-                const offer = await this.peerConnection.createOffer();
-                await this.peerConnection.setLocalDescription(offer);
-                this.signaling?.send({ type: 'offer', sdp: this.peerConnection.localDescription?.sdp });
+            this.localStream = await navigator.mediaDevices.getUserMedia({
+                audio: {
+                    echoCancellation: true,
+                    noiseSuppression: true,
+                    autoGainControl: true,
+                },
+                video: false,
+            });
+            const audioTrack = this.localStream.getAudioTracks()[0];
+            if (this.peerConnection && audioTrack) {
+                if (this.audioTransceiver) {
+                    this.audioTransceiver.direction = 'sendrecv';
+                    if (this.audioTransceiver.sender) {
+                        await this.audioTransceiver.sender.replaceTrack(audioTrack);
+                    }
+                }
+                else {
+                    const transceiver = this.peerConnection
+                        .getTransceivers()
+                        .find((t) => t.receiver.track.kind === 'audio');
+                    if (transceiver) {
+                        this.audioTransceiver = transceiver;
+                        this.audioTransceiver.direction = 'sendrecv';
+                        await this.audioTransceiver.sender.replaceTrack(audioTrack);
+                    }
+                    else {
+                        this.peerConnection.addTrack(audioTrack, this.localStream);
+                    }
+                }
+                // The direction change (or addTrack) fires negotiationneeded, which sends the offer.
             }
+            return this.localStream;
         }
         catch (err) {
-            console.error('Failed to publish audio:', err);
+            console.error('[Room] Failed to publish audio:', err);
             this.emit('error', err);
+            throw err;
         }
     }
+    /**
+     * Sets microphone mute state.
+     */
+    setMuted(muted) {
+        if (this.localStream) {
+            this.localStream.getAudioTracks().forEach((track) => {
+                track.enabled = !muted;
+            });
+            this.emit('muteChanged', muted);
+        }
+    }
+    /**
+     * Returns whether the microphone is currently muted.
+     */
+    isMuted() {
+        if (!this.localStream)
+            return true;
+        const tracks = this.localStream.getAudioTracks();
+        return tracks.length === 0 || !tracks[0].enabled;
+    }
+    /**
+     * Gets the active local media stream.
+     */
+    getLocalStream() {
+        return this.localStream;
+    }
+    /**
+     * Gets all subscribed remote audio streams.
+     */
+    getRemoteStreams() {
+        return Array.from(this.remoteStreams.values());
+    }
     disconnect() {
         if (this.localStream) {
-            this.localStream.getTracks().forEach(track => track.stop());
+            this.localStream.getTracks().forEach((track) => track.stop());
+            this.localStream = null;
         }
         if (this.peerConnection) {
+            this.peerConnection.onicecandidate = null;
+            this.peerConnection.ontrack = null;
+            this.peerConnection.onconnectionstatechange = null;
             this.peerConnection.close();
             this.peerConnection = null;
         }
+        this.audioTransceiver = null;
+        this.candidateQueue = [];
+        this.remoteStreams.clear();
         if (this.signaling) {
             this.signaling.close();
             this.signaling = null;

package/dist/signaling.d.ts

@@ -3,8 +3,19 @@ import { SignalMessage } from './types';
 export declare class SignalingClient extends EventEmitter {
     private ws;
     private url;
+    private outbox;
+    private isClosedManually;
+    private retryAttempts;
+    private readonly maxRetries;
+    private reconnectTimer;
+    private heartbeatTimer;
     constructor(nodeAddress: string, ticket: string);
+    private buildUrl;
     connect(): void;
     send(message: SignalMessage): void;
+    private flushOutbox;
+    private startHeartbeat;
+    private stopHeartbeat;
+    private cleanupSocket;
     close(): void;
 }