videomail-client 16.2.0 → 16.4.0

package/dist/esm/index.js

@@ -3620,7 +3620,7 @@ const constants = {
     }
 };
 var package_namespaceObject = {
-    rE: "16.2.0"
+    rE: "16.4.0"
 };
 function isAudioEnabled(options) {
     return Boolean(options.audio.enabled);
@@ -7740,6 +7740,49 @@ const MEDIA_EVENTS = [
     "volumechange"
 ];
 const mediaEvents = MEDIA_EVENTS;
+const AUDIO_RECORDER_PROCESSOR_NAME = "videomail-pcm-recorder";
+function registerAudioRecorderProcessor(name, Processor, register) {
+    class PcmRecorderProcessor extends Processor {
+        process(inputs) {
+            if (this.recordingId < 0) return false;
+            const channel = inputs[0]?.[0];
+            if (!channel || 0 === this.recordingId) return true;
+            let inputOffset = 0;
+            while(inputOffset < channel.length){
+                const count = Math.min(this.samples.length - this.offset, channel.length - inputOffset);
+                this.samples.set(channel.subarray(inputOffset, inputOffset + count), this.offset);
+                this.offset += count;
+                inputOffset += count;
+                if (this.offset === this.samples.length) {
+                    const samples = this.samples;
+                    this.port.postMessage({
+                        recordingId: this.recordingId,
+                        samples
+                    }, [
+                        samples.buffer
+                    ]);
+                    this.samples = new Float32Array(this.offset);
+                    this.offset = 0;
+                }
+            }
+            return true;
+        }
+        constructor(options){
+            super();
+            this.samples = new Float32Array(options.processorOptions.bufferSize);
+            this.offset = 0;
+            this.recordingId = 0;
+            this.port.onmessage = (event)=>{
+                const recordingId = event.data;
+                if ("number" != typeof recordingId) return;
+                if (!Number.isSafeInteger(recordingId) || recordingId < -1) return;
+                this.recordingId = recordingId;
+                this.offset = 0;
+            };
+        }
+    }
+    register(name, PcmRecorderProcessor);
+}
 function AudioRecorder_define_property(obj, key, value) {
     if (key in obj) Object.defineProperty(obj, key, {
         value: value,
@@ -7751,40 +7794,113 @@ function AudioRecorder_define_property(obj, key, value) {
     return obj;
 }
 const CHANNELS = 1;
-function getAudioContextClass() {
-    return window.AudioContext;
-}
-class AudioRecorder {
-    hasAudioContext() {
-        return Boolean(getAudioContextClass()) && Boolean(this.getAudioContext());
-    }
+const DEFAULT_BUFFER_SIZE = 2048;
+const MIN_BUFFER_SIZE = 256;
+const MAX_BUFFER_SIZE = 16384;
+class AudioRecorder extends util_Despot {
     getAudioContext() {
-        if (!this.vcAudioContext) {
-            const AudioContext = getAudioContextClass();
-            this.vcAudioContext = new AudioContext();
-        }
+        if (!this.vcAudioContext) this.vcAudioContext = new window.AudioContext();
         return this.vcAudioContext;
     }
-    onAudioProcess(e, cb) {
+    getBufferSize() {
+        const configuredSize = this.options.audio.bufferSize;
+        const bufferSize = "auto" === configuredSize ? DEFAULT_BUFFER_SIZE : configuredSize;
+        if (void 0 === bufferSize) throw error_createError({
+            message: "Audio buffer size is required.",
+            options: this.options
+        });
+        const validBufferSize = Number.isInteger(bufferSize) && bufferSize >= MIN_BUFFER_SIZE && bufferSize <= MAX_BUFFER_SIZE && is_power_of_two(bufferSize);
+        if (!validBufferSize) throw error_createError({
+            message: `Audio buffer size must be a power of two between ${MIN_BUFFER_SIZE} and ${MAX_BUFFER_SIZE}. The current buffer size is ${bufferSize}.`,
+            options: this.options
+        });
+        return bufferSize;
+    }
+    validateVolume() {
+        const { volume } = this.options.audio;
+        const validVolume = Number.isFinite(volume) && volume >= 0 && volume <= 1;
+        if (!validVolume) throw error_createError({
+            message: `Audio volume must be between zero and one. The current volume is ${volume}.`,
+            options: this.options
+        });
+    }
+    revokeModuleUrl() {
+        if (this.moduleUrl) {
+            URL.revokeObjectURL(this.moduleUrl);
+            this.moduleUrl = void 0;
+        }
+    }
+    async loadWorklet(context) {
+        const source = `(${registerAudioRecorderProcessor.toString()})(${JSON.stringify(AUDIO_RECORDER_PROCESSOR_NAME)}, AudioWorkletProcessor, registerProcessor);`;
+        const moduleUrl = URL.createObjectURL(new Blob([
+            source
+        ], {
+            type: "text/javascript"
+        }));
+        this.moduleUrl = moduleUrl;
+        try {
+            await context.audioWorklet.addModule(moduleUrl);
+        } finally{
+            if (this.moduleUrl === moduleUrl) this.revokeModuleUrl();
+        }
+    }
+    onAudioProcess(event) {
+        const callback = this.audioCallback;
+        if (!callback) return;
         if (!this.userMedia.isRecording() || this.userMedia.isPaused()) return;
-        const float32Array = e.inputBuffer.getChannelData(0);
-        cb(new audio_sample(float32Array));
+        const message = event.data;
+        if ("object" != typeof message || null === message) return;
+        if (!("recordingId" in message) || message.recordingId !== this.recordingId) return;
+        if (!("samples" in message) || !(message.samples instanceof Float32Array)) return;
+        if (0 === message.samples.length) return;
+        callback(new audio_sample(message.samples));
     }
     disconnect() {
-        if (this.scriptProcessor) {
-            this.scriptProcessor.onaudioprocess = null;
-            this.scriptProcessor.disconnect();
+        if (this.worklet) {
+            this.worklet.port.onmessage = null;
+            this.worklet.onprocessorerror = null;
+            this.worklet.port.postMessage(-1);
+            this.worklet.port.close();
+            this.worklet.disconnect();
+            this.worklet = void 0;
         }
         this.audioInput?.disconnect();
         this.volume?.disconnect();
+        this.audioInput = void 0;
+        this.volume = void 0;
+        this.audioCallback = void 0;
+        this.recordingId = 0;
     }
-    init(localMediaStream) {
+    async init(localMediaStream) {
         this.options.logger.debug("AudioRecorder: init()");
+        const initializationId = ++this.initializationId;
         this.disconnect();
-        const volume = this.getAudioContext().createGain();
-        this.volume = volume;
+        const bufferSize = this.getBufferSize();
+        this.validateVolume();
+        if ("function" != typeof window.AudioContext) throw error_createError({
+            message: "Audio recording requires Web Audio support.",
+            options: this.options
+        });
+        const context = this.getAudioContext();
+        if (void 0 === context.audioWorklet || "function" != typeof window.AudioWorkletNode) throw error_createError({
+            message: "Audio recording requires AudioWorklet support in a secure context.",
+            options: this.options
+        });
+        const workletReady = this.workletReady ??= this.loadWorklet(context);
+        try {
+            await workletReady;
+        } catch (exc) {
+            if (this.workletReady === workletReady) this.workletReady = void 0;
+            if (initializationId !== this.initializationId) return;
+            throw error_createError({
+                message: "Unable to load the audio recorder worklet. Check AudioWorklet and Content Security Policy support for blob: scripts.",
+                cause: exc,
+                options: this.options
+            });
+        }
+        if (initializationId !== this.initializationId) return;
         try {
-            this.audioInput = this.getAudioContext().createMediaStreamSource(localMediaStream);
+            this.audioInput = context.createMediaStreamSource(localMediaStream);
         } catch (exc) {
             throw error_createError({
                 message: "Webcam has no audio",
@@ -7792,38 +7908,69 @@ class AudioRecorder {
                 options: this.options
             });
         }
-        let { bufferSize } = this.options.audio;
-        if ("auto" === bufferSize) bufferSize = util_getBrowser(this.options).isFirefox() ? 512 : 2048;
-        if (void 0 === bufferSize || !is_power_of_two(bufferSize)) throw error_createError({
-            message: `Audio buffer size must be a power of two. The current buffer size is ${bufferSize}.`,
-            options: this.options
-        });
-        if (!Number.isFinite(this.options.audio.volume) || this.options.audio.volume < 0 || this.options.audio.volume > 1) throw error_createError({
-            message: `Audio volume must be between zero and one. The current volume is ${this.options.audio.volume}.`,
-            options: this.options
-        });
-        volume.gain.value = this.options.audio.volume;
-        this.scriptProcessor = this.getAudioContext().createScriptProcessor(bufferSize, CHANNELS, CHANNELS);
-        this.audioInput.connect(volume);
-        volume.connect(this.scriptProcessor);
-        this.scriptProcessor.connect(this.getAudioContext().destination);
+        try {
+            this.volume = context.createGain();
+            this.volume.gain.value = this.options.audio.volume;
+            this.worklet = new window.AudioWorkletNode(context, AUDIO_RECORDER_PROCESSOR_NAME, {
+                channelCount: CHANNELS,
+                channelCountMode: "explicit",
+                outputChannelCount: [
+                    CHANNELS
+                ],
+                processorOptions: {
+                    bufferSize
+                }
+            });
+            this.worklet.port.onmessage = (event)=>{
+                this.onAudioProcess(event);
+            };
+            this.worklet.onprocessorerror = ()=>{
+                this.stop();
+                this.emit("ERROR", {
+                    err: error_createError({
+                        message: "Audio recording processor failed.",
+                        options: this.options
+                    })
+                });
+            };
+            this.audioInput.connect(this.volume);
+            this.volume.connect(this.worklet);
+            this.worklet.connect(context.destination);
+        } catch (exc) {
+            this.disconnect();
+            throw error_createError({
+                message: "Failed to connect the audio recorder.",
+                exc,
+                options: this.options
+            });
+        }
     }
     record(cb) {
         this.options.logger.debug("AudioRecorder: record()");
-        if (this.scriptProcessor) this.scriptProcessor.onaudioprocess = (e)=>{
-            this.onAudioProcess(e, cb);
-        };
+        this.audioCallback = cb;
+        this.updateRecordingState();
+    }
+    updateRecordingState() {
+        if (!this.worklet) return;
+        const shouldRecord = Boolean(this.audioCallback) && this.userMedia.isRecording() && !this.userMedia.isPaused();
+        this.recordingId = shouldRecord ? ++this.recordingSequence : 0;
+        this.worklet.port.postMessage(this.recordingId);
     }
     async resume() {
-        const context = this.getAudioContext();
+        const context = this.vcAudioContext;
+        if (!context) return;
         if ("running" !== context.state) await context.resume();
     }
     stop() {
         this.options.logger.debug("AudioRecorder: stop()");
+        this.initializationId++;
         this.disconnect();
-        if (this.hasAudioContext()) this.getAudioContext().close().then(()=>{
+        this.revokeModuleUrl();
+        const context = this.vcAudioContext;
+        this.vcAudioContext = void 0;
+        this.workletReady = void 0;
+        if (context && "closed" !== context.state) context.close().then(()=>{
             this.options.logger.debug("AudioRecorder: audio context is closed");
-            this.vcAudioContext = void 0;
         }).catch((err)=>{
             if (err instanceof Error) return void this.options.logger.error(error_createError({
                 err,
@@ -7833,17 +7980,11 @@ class AudioRecorder {
         });
     }
     getSampleRate() {
-        if (this.hasAudioContext()) return this.getAudioContext().sampleRate;
+        if ("function" == typeof window.AudioContext) return this.getAudioContext().sampleRate;
         return -1;
     }
     constructor(userMedia, options){
-        AudioRecorder_define_property(this, "scriptProcessor", void 0);
-        AudioRecorder_define_property(this, "audioInput", void 0);
-        AudioRecorder_define_property(this, "volume", void 0);
-        AudioRecorder_define_property(this, "vcAudioContext", void 0);
-        AudioRecorder_define_property(this, "userMedia", void 0);
-        AudioRecorder_define_property(this, "options", void 0);
-        this.options = options;
+        super("AudioRecorder", options), AudioRecorder_define_property(this, "userMedia", void 0), AudioRecorder_define_property(this, "worklet", void 0), AudioRecorder_define_property(this, "audioInput", void 0), AudioRecorder_define_property(this, "volume", void 0), AudioRecorder_define_property(this, "vcAudioContext", void 0), AudioRecorder_define_property(this, "workletReady", void 0), AudioRecorder_define_property(this, "moduleUrl", void 0), AudioRecorder_define_property(this, "audioCallback", void 0), AudioRecorder_define_property(this, "initializationId", 0), AudioRecorder_define_property(this, "recordingSequence", 0), AudioRecorder_define_property(this, "recordingId", 0);
         this.userMedia = userMedia;
     }
 }
@@ -7989,27 +8130,33 @@ class UserMedia extends util_Despot {
         const fireCallbacks = ()=>{
             const readyState = this.rawVisualUserMedia?.readyState;
             this.options.logger.debug(`UserMedia: fireCallbacks(readyState=${readyState}, onPlayReached=${this.onPlayReached}, onLoadedMetaDataReached=${this.onLoadedMetaDataReached})`);
-            if (this.onPlayReached && this.onLoadedMetaDataReached) if (this.audioRecorder) try {
-                this.audioRecorder.init(localMediaStream);
-                const onAudioReady = ()=>{
-                    if (initializationId !== this.initializationId || this.currentVisualStream !== localMediaStream) return;
-                    if (this.recording) this.audioRecord(audioCallback);
-                    else this.on("SENDING_FIRST_FRAME", ()=>{
-                        this.audioRecord(audioCallback);
-                    });
-                    this.clearInitializationListeners?.();
-                    videoCallback();
+            if (this.onPlayReached && this.onLoadedMetaDataReached) if (this.audioRecorder) {
+                const initializeAudio = async (audioRecorder)=>{
+                    try {
+                        await audioRecorder.init(localMediaStream);
+                        if (initializationId !== this.initializationId || this.currentVisualStream !== localMediaStream) return;
+                        const onAudioReady = ()=>{
+                            if (initializationId !== this.initializationId || this.currentVisualStream !== localMediaStream) return;
+                            if (this.recording) this.audioRecord(audioCallback);
+                            else this.on("SENDING_FIRST_FRAME", ()=>{
+                                this.audioRecord(audioCallback);
+                            });
+                            this.clearInitializationListeners?.();
+                            videoCallback();
+                        };
+                        if (params?.recovering) await audioRecorder.resume();
+                        onAudioReady();
+                    } catch (exc) {
+                        if (initializationId !== this.initializationId) return;
+                        unloadAllEventListeners();
+                        this.stop(void 0, {
+                            preserveRecordingState: Boolean(params?.recovering)
+                        });
+                        endedEarlyCallback(exc);
+                    }
                 };
-                if (params?.recovering) this.audioRecorder.resume().then(onAudioReady).catch(endedEarlyCallback);
-                else onAudioReady();
-            } catch (exc) {
-                unloadAllEventListeners();
-                this.stop(void 0, {
-                    preserveRecordingState: Boolean(params?.recovering)
-                });
-                endedEarlyCallback(exc);
-            }
-            else {
+                initializeAudio(this.audioRecorder);
+            } else {
                 this.clearInitializationListeners?.();
                 videoCallback();
             }
@@ -8166,15 +8313,18 @@ class UserMedia extends util_Despot {
     }
     pause() {
         this.paused = true;
+        this.audioRecorder?.updateRecordingState();
     }
     isPaused() {
         return this.paused;
     }
     resume() {
         this.paused = false;
+        this.audioRecorder?.updateRecordingState();
     }
     record() {
         this.recording = true;
+        this.audioRecorder?.updateRecordingState();
     }
     isRecording() {
         return this.recording;

package/dist/esm/util/html/media/AudioRecorder.d.ts

@@ -1,21 +1,31 @@
 import AudioSample from "audio-sample";
-import { VideomailClientOptions } from "../../../types/options.js";
-import UserMedia from "../../../wrappers/visuals/userMedia.js";
+import type { VideomailClientOptions } from "../../../types/options.js";
+import type UserMedia from "../../../wrappers/visuals/userMedia.js";
+import Despot from "../../Despot.js";
 export type AudioProcessCB = (audioSample: AudioSample) => void;
-declare class AudioRecorder {
-    private scriptProcessor?;
+declare class AudioRecorder extends Despot {
+    private readonly userMedia;
+    private worklet?;
     private audioInput?;
     private volume?;
     private vcAudioContext?;
-    private readonly userMedia;
-    private readonly options;
-    constructor(userMedia: UserMedia, options: VideomailClientOptions);
-    private hasAudioContext;
+    private workletReady?;
+    private moduleUrl?;
+    private audioCallback?;
+    private initializationId;
+    private recordingSequence;
+    private recordingId;
+    constructor(userMedia: Pick<UserMedia, "isRecording" | "isPaused">, options: VideomailClientOptions);
     private getAudioContext;
+    private getBufferSize;
+    private validateVolume;
+    private revokeModuleUrl;
+    private loadWorklet;
     private onAudioProcess;
     private disconnect;
-    init(localMediaStream: MediaStream): void;
+    init(localMediaStream: MediaStream): Promise<void>;
     record(cb: AudioProcessCB): void;
+    updateRecordingState(): void;
     resume(): Promise<void>;
     stop(): void;
     getSampleRate(): number;

package/dist/esm/util/html/media/AudioRecorderWorklet.d.ts

@@ -0,0 +1,17 @@
+export declare const AUDIO_RECORDER_PROCESSOR_NAME = "videomail-pcm-recorder";
+export interface AudioRecorderProcessorOptions {
+    processorOptions: {
+        bufferSize: number;
+    };
+}
+export interface AudioRecorderProcessor {
+    process(inputs: readonly (readonly Float32Array[])[], outputs: readonly (readonly Float32Array[])[]): boolean;
+}
+interface AudioRecorderProcessorBase {
+    readonly port: {
+        onmessage: ((event: MessageEvent<unknown>) => void) | null;
+        postMessage: (message: unknown, transfer: Transferable[]) => void;
+    };
+}
+export default function registerAudioRecorderProcessor(name: string, Processor: new () => AudioRecorderProcessorBase, register: (processorName: string, processor: new (options: AudioRecorderProcessorOptions) => AudioRecorderProcessor) => void): void;
+export {};