twilio-video 2.22.2 → 2.23.0

package/CHANGELOG.md

@@ -2,6 +2,50 @@ The Twilio Programmable Video SDKs use [Semantic Versioning](http://www.semver.o
 
 **Version 1.x reached End of Life on September 8th, 2021.** See the changelog entry [here](https://www.twilio.com/changelog/end-of-life-complete-for-unsupported-versions-of-the-programmable-video-sdk). Support for the 1.x version ended on December 4th, 2020.
 
+2.23.0 (July 28, 2022)
+======================
+
+New Features
+------------
+
+- This release adds private beta support for 3rd party noise cancellation solution. You need to host twilio approved 3rd party plugin on your web server to enable noise cancellation. Please fill out [this form](https://forms.gle/eeFyoGJj1mgMrxN88) to request access to the 3rd party plugin.
+
+Once you get the access to the plugin, You can install it from npm with:
+```
+npm install <noise_cancellation_plugin>
+```
+
+Once installed, you need to host the contents of `./node_modules/<noise_cancellation_plugin>/dist/` from your web server. We recommend that you add plugin version number to the hosted path to ensure that browser does not use [stale version](https://www.keycdn.com/support/what-is-cache-busting) when its updated. You need to pass the path to the hosted files to `twilio-video` sdk when creating audio track as shown in the example below. The example below assumes that you have hosted the files at `/noise_cancellation_plugin/1.0.0/dist` on your web server.
+
+```ts
+const { connect, createLocalAudioTrack } = require('twilio-video');
+
+// create a local audio track and have it use
+// @twilio/krisp-audio-plugin for noise cancellation processing.
+const localAudioTrack = await Video.createLocalAudioTrack({
+  noiseCancellationOptions: {
+    vendor: 'krisp',
+    sdkAssetsPath: '/noise_cancellation_plugin/1.0.0/dist'
+  }
+});
+
+// publish the track to a room
+const room = await connect( token, {
+  tracks: [localAudioTrack]
+  // ... any other connect options
+});
+
+// you can enable/disable noise cancellation at runtime
+// using noiseCancellation interface exposed by localAudioTrack
+function updateNoiseCancellation(enable: boolean) {
+  const noiseCancellation = localAudioTrack.noiseCancellation;
+
+  if (noiseCancellation) {
+    enable ? noiseCancellation.enable() : noiseCancellation.disable();
+  }
+}
+
+```
 2.22.2 (July 25, 2022)
 ======================
 
@@ -9,7 +53,7 @@ Changes
 -------
 
 - `isSupported` flag now returns `false` if the browser does not support the Unified Plan SDP format. (VIDEO-10307)
-  
+
   The following is a list of browsers with Unified Plan as the default SDP format.
   - Chrome 72+
   - Safari 12.1+

package/README.md

@@ -74,7 +74,7 @@ Releases of twilio-video.js are hosted on a CDN, and you can include these
 directly in your web app using a <script> tag.
 
 ```html
-<script src="//sdk.twilio.com/js/video/releases/2.22.2/twilio-video.min.js"></script>
+<script src="//sdk.twilio.com/js/video/releases/2.23.0/twilio-video.min.js"></script>
 
 ```
 

package/assets/noisecancellation/rnnoise/rnnoise_processor.js

@@ -0,0 +1,103 @@
+'use strict';
+let rnnoiseExports = null;
+let heapFloat32;
+let processCount = 0;
+class RNNNoiseProcessor extends AudioWorkletProcessor {
+  constructor(options) {
+    super(Object.assign(Object.assign({}, options), { numberOfInputs: 1, numberOfOutputs: 1, outputChannelCount: [1] }));
+    if (!rnnoiseExports) {
+      rnnoiseExports = new WebAssembly.Instance(options.processorOptions.module).exports;
+      heapFloat32 = new Float32Array(rnnoiseExports.memory.buffer);
+    }
+    // enable
+    // disable
+    // destroy
+    // enableLogging
+    // disableLogging
+    this.logging = true;
+    this.keepAlive = true;
+    if (options.processorOptions.activeInitially) {
+      this.log('activeInitially');
+      this.state = rnnoiseExports.newState();
+    } else {
+      this.log('NOT activeInitially');
+      this.state = null;
+    }
+
+    this.port.onmessage = ({ data: command }) => {
+      let vadProb = 0;
+      this.log('handling command: ', command);
+      switch (command) {
+        case 'enable':
+          if (!this.state) {
+            this.state = rnnoiseExports.newState();
+          }
+          vadProb = rnnoiseExports.getVadProb(this.state);
+          break;
+        case 'disable':
+          if (this.state) {
+            rnnoiseExports.deleteState(this.state);
+            this.state = null;
+          }
+          break;
+        case 'destroy':
+          if (this.state) {
+            rnnoiseExports.deleteState(this.state);
+            this.state = null;
+          }
+          this.keepAlive = false;
+          break;
+        case 'enableLogging':
+          this.logging = true;
+          this.log('logging Enabled');
+          break;
+        case 'disableLogging':
+          this.log('Will disable logging');
+          this.logging = false;
+          break;
+        default:
+          // eslint-disable-next-line no-console
+          console.warn('unknown command: ', command);
+      }
+      if (command === 'enable' && this.state === null) {
+        this.log('processor creating state again');
+        this.state = rnnoiseExports.newState();
+      }
+      this.port.postMessage({
+        vadProb,
+        isEnabled: this.state !== null,
+        isAlive: this.keepAlive
+      });
+    };
+  }
+
+  log(...messages) {
+    if (this.logging) {
+      // eslint-disable-next-line no-console
+      console.log('rnnoise_processor:', ...messages);
+    }
+  }
+
+  process(inputs, outputs, parameters) {
+    if (this.state && inputs && inputs.length > 0 && inputs[0].length > 0) {
+      heapFloat32.set(inputs[0][0], rnnoiseExports.getInput(this.state) / 4);
+      const o = outputs[0][0];
+      const ptr4 = rnnoiseExports.pipe(this.state, o.length) / 4;
+      if (ptr4) {
+        o.set(heapFloat32.subarray(ptr4, ptr4 + o.length));
+      }
+    } else {
+      // rnnoise is turned off.
+      // eslint-disable-next-line no-lonely-if
+      if (inputs && inputs.length > 0 && inputs[0].length > 0) {
+        outputs[0][0].set(inputs[0][0]);
+      }
+    }
+    processCount++;
+    if (processCount % 1111 === 0) {
+      this.log(`${processCount}: RNNoise ${this.state ? 'enabled' : 'disabled'}`);
+    }
+    return this.keepAlive;
+  }
+}
+registerProcessor('rnnoise', RNNNoiseProcessor);

package/assets/noisecancellation/rnnoise/rnnoise_sdk.mjs

@@ -0,0 +1,150 @@
+// Author: Makarand Patwardhan
+// Implements interface to interact with twilio-video SDK
+
+/* eslint-disable no-console */
+async function fetchAndCompileWebAssemblyModule(moduleUrl) {
+  const response = await fetch(moduleUrl);
+  const buffer = await response.arrayBuffer();
+  const mod = await WebAssembly.compile(buffer);
+  return mod;
+}
+class RNNoiseNode extends AudioWorkletNode {
+  constructor(context) {
+    super(context, 'rnnoise', {
+      channelCountMode: 'explicit',
+      channelCount: 1,
+      channelInterpretation: 'speakers',
+      numberOfInputs: 1,
+      numberOfOutputs: 1,
+      outputChannelCount: [1],
+      processorOptions: {
+        module: RNNoiseNode.webModule,
+        activeInitially: false,
+      },
+    });
+    this._vadProb = 0;
+    this._isEnabled = false; // match what we sent in processorOptions.activeInitially.
+    this._isAlive = false;
+    console.log('RNNoiseNode constructor');
+    this.port.onmessage = ({ data }) => {
+      this._isEnabled = data.isEnabled;
+      this._isAlive = data.isAlive;
+      if (data.vadProb) {
+        this._vadProb = data.vadProb;
+        if (this._onUpdate) {
+          this._onUpdate();
+        }
+      }
+    };
+  }
+  static async register(context, webAssemblyUrl, processorUrl) {
+    RNNoiseNode.webModule = await fetchAndCompileWebAssemblyModule(webAssemblyUrl);
+    await context.audioWorklet.addModule(processorUrl);
+  }
+  onUpdate(onupdate) {
+    this._onUpdate = onupdate;
+  }
+  getVadProb() {
+    return this._vadProb;
+  }
+  getIsEnabled() {
+    return this._isEnabled;
+  }
+  enable() {
+    this.port.postMessage('enable');
+    this._isEnabled = true;
+  }
+  disable() {
+    this.port.postMessage('disable');
+    this._isEnabled = false;
+  }
+  destroy() {
+    this.port.postMessage('destroy');
+  }
+  setLogging(enabled) {
+    this.port.postMessage(enabled ? 'enableLogging' : 'disableLogging');
+  }
+}
+RNNoiseNode.webModule = null;
+let initialized = false;
+let audioContext = null;
+let connected = false;
+let rnnoiseNode = null;
+let sourceNode = null;
+let destinationNode = null;
+let inputStream = null;
+
+const init = async ({ rootDir }) => {
+  console.log('rnnoise.init, was initialized = ', initialized);
+  audioContext = new AudioContext();
+  const wasmPath = rootDir + '/rnnoise-processor.wasm';
+  const processorJSPath = rootDir + '/rnnoise_processor.js';
+  console.log({ wasmPath, processorJSPath });
+  await RNNoiseNode.register(audioContext, wasmPath, processorJSPath);
+  initialized = true;
+};
+
+const isInitialized = () => initialized;
+
+const connect = stream => {
+  if (!isInitialized()) {
+    throw new Error('you must call init before connect');
+  }
+  sourceNode = audioContext.createMediaStreamSource(stream);
+  if (!rnnoiseNode) {
+    rnnoiseNode = new RNNoiseNode(audioContext);
+  }
+  destinationNode = audioContext.createMediaStreamDestination();
+  sourceNode.connect(rnnoiseNode);
+  rnnoiseNode.connect(destinationNode);
+  const outputStream = destinationNode.stream;
+  connected = true;
+  inputStream = stream;
+  rnnoiseNode.enable();
+  return outputStream;
+};
+
+const isConnected = () => connected;
+
+const disconnect = () => {
+  if (connected) {
+    connected = false;
+    rnnoiseNode.disable();
+    rnnoiseNode.disconnect();
+    sourceNode.disconnect();
+    destinationNode.disconnect();
+    inputStream.getTracks().forEach(track => track.stop());
+  }
+};
+
+const enable = () => connected && rnnoiseNode.enable();
+const isEnabled = () => connected && rnnoiseNode.getIsEnabled();
+const disable = () => connected && rnnoiseNode.disable();
+const destroy = () => {
+  initialized = false;
+  disconnect();
+  if (rnnoiseNode) {
+    rnnoiseNode.destroy();
+    rnnoiseNode = null;
+  }
+};
+
+const setLogging = enable => {
+  if (rnnoiseNode) {
+    rnnoiseNode.setLogging(enable);
+  }
+};
+
+export default {
+  init,
+  isInitialized,
+  connect,
+  isConnected,
+  disconnect,
+  enable,
+  isEnabled,
+  disable,
+  destroy,
+  setLogging,
+};
+