twilio-video 2.22.2 → 2.23.0

package/lib/media/track/noisecancellationimpl.ts

@@ -0,0 +1,143 @@
+'use strict';
+
+import { NoiseCancellation, NoiseCancellationOptions, NoiseCancellationVendor } from '../../../tsdef/types';
+import { AudioProcessor } from '../../../tsdef/AudioProcessor';
+import { createNoiseCancellationAudioProcessor } from '../../noisecancellationadapter';
+const Log = require('../../util/log');
+
+/**
+ * {@link NoiseCancellation} interface provides methods to control noise cancellation at runtime.
+ * This interface is exposed on {@link LocalAudioTrack} property `noiseCancellation`. It is available only when
+ * {@link NoiseCancellationOptions} are specified when creating a {@link LocalAudioTrack}
+ * @alias NoiseCancellation
+ * @interface
+ *
+ * @example
+ * const { connect, createLocalAudioTrack } = require('twilio-video');
+ *
+ * // create a local audio track and have it use
+ * // @twilio/krisp-audio-plugin for noise cancellation processing.
+ * const localAudioTrack = await Video.createLocalAudioTrack({
+ *   noiseCancellationOptions: {
+ *     vendor: 'krisp',
+ *     sdkAssetsPath: '/twilio-krisp-audio-plugin/1.0.0/dist'
+ *   }
+ * });
+ *
+ * // publish the track to a room
+ * const room = await connect( token, {
+ *   tracks: [localAudioTrack]
+ *   // ... any other connect options
+ * });
+ *
+ * // you can enable/disable noise cancellation at runtime
+ * // using noiseCancellation interface exposed by localAudioTrack
+ * function updateNoiseCancellation(enable: boolean) {
+ *   const noiseCancellation = localAudioTrack.noiseCancellation;
+ *
+ *   if (noiseCancellation) {
+ *     enable ? noiseCancellation.enable() : noiseCancellation.disable();
+ *   }
+ * }
+ */
+export class NoiseCancellationImpl implements NoiseCancellation {
+  private _processor: AudioProcessor;
+  private _sourceTrack: MediaStreamTrack;
+  private _disabledPermanent: boolean;
+  constructor(processor: AudioProcessor, originalTrack: MediaStreamTrack) {
+    this._processor = processor;
+    this._sourceTrack = originalTrack;
+    this._disabledPermanent = false;
+  }
+
+  /**
+   * Identifies the vendor
+   * @type {NoiseCancellationVendor}
+   */
+  get vendor(): NoiseCancellationVendor {
+    return this._processor.vendor;
+  }
+
+  /**
+   * Underlying MediaStreamTrack
+   * @type {MediaStreamTrack}
+   */
+  get sourceTrack(): MediaStreamTrack {
+    return this._sourceTrack;
+  }
+
+  /**
+   * Set to true if noise cancellation is currently enabled
+   * @type {boolean}
+   */
+  get isEnabled(): boolean {
+    return this._processor.isEnabled();
+  }
+
+  /**
+   * Enables noise cancellation
+   * @returns {Promise<void>} a promise that resolves when operation is complete.
+   */
+  enable() : Promise<void> {
+    if (this._disabledPermanent) {
+      throw new Error(`${this.vendor} noise cancellation is disabled permanently for this track`);
+    }
+
+    this._processor.enable();
+    return Promise.resolve();
+  }
+
+  /**
+   * Disables noise cancellation
+   * @returns {Promise<void>} a promise that resolves when operation is complete.
+   */
+  disable() : Promise<void> {
+    this._processor.disable();
+    return Promise.resolve();
+  }
+
+  /**
+   * @private
+   */
+  async reacquireTrack(reacquire: () => Promise<MediaStreamTrack>) : Promise<MediaStreamTrack>  {
+    const processorWasEnabled = this._processor.isEnabled();
+    this._processor.disconnect();
+
+    const track = await reacquire();
+    this._sourceTrack = track;
+
+    const processedTrack = await this._processor.connect(track);
+    if (processorWasEnabled) {
+      this._processor.enable();
+    } else {
+      this._processor.disable();
+    }
+    return processedTrack;
+  }
+
+  /**
+   * @private
+   */
+  disablePermanently(): Promise<void> {
+    this._disabledPermanent = true;
+    return this.disable();
+  }
+}
+
+
+export async function applyNoiseCancellation(
+  mediaStreamTrack: MediaStreamTrack,
+  noiseCancellationOptions: NoiseCancellationOptions,
+  log: typeof Log
+) : Promise<{ cleanTrack: MediaStreamTrack, noiseCancellation?: NoiseCancellation }> {
+  try {
+    const processor = await createNoiseCancellationAudioProcessor(noiseCancellationOptions, log);
+    const cleanTrack = processor.connect(mediaStreamTrack);
+    const noiseCancellation = new NoiseCancellationImpl(processor, mediaStreamTrack);
+    return { cleanTrack, noiseCancellation };
+  } catch (ex) {
+    // in case of failures to load noise cancellation library just return original media stream.
+    log.warn(`Failed to create noise cancellation. Returning normal audio track: ${ex}`);
+    return { cleanTrack: mediaStreamTrack };
+  }
+}

package/lib/noisecancellationadapter.ts

@@ -0,0 +1,93 @@
+/* eslint-disable no-console */
+'use strict';
+import { AudioProcessor } from '../tsdef/AudioProcessor';
+import { NoiseCancellationOptions } from '../tsdef/types';
+const Log = require('./util/log');
+
+const dynamicImport = require('./dynamicImport');
+const KRISP_PLUGIN_FILE = 'krispsdk.mjs';
+const RNNOISE_PLUGIN_FILE = 'rnnoise_sdk.mjs';
+
+// AudioProcessor assumes following interface from the Plugin
+interface NoiseCancellationPlugin {
+  init(options: { rootDir: string }): Promise<void>;
+  isInitialized(): boolean;
+  isConnected(): boolean;
+  isEnabled(): boolean
+  connect(input: MediaStream): MediaStream;
+  disconnect(): void;
+  enable(): void;
+  disable(): void;
+  destroy(): void;
+  setLogging(enable: boolean):void;
+}
+
+let audioProcessors = new Map<string, AudioProcessor>();
+export async function createNoiseCancellationAudioProcessor(
+  noiseCancellationOptions: NoiseCancellationOptions,
+  log: typeof Log
+) : Promise<AudioProcessor> {
+  let audioProcessor = audioProcessors.get(noiseCancellationOptions.vendor);
+  if (!audioProcessor) {
+    let sdkFilePath: string;
+    let rootDir = noiseCancellationOptions.sdkAssetsPath;
+    switch (noiseCancellationOptions.vendor) {
+      case 'krisp':
+        sdkFilePath = `${rootDir}/${KRISP_PLUGIN_FILE}`;
+        break;
+      case 'rnnoise':
+        sdkFilePath = `${rootDir}/${RNNOISE_PLUGIN_FILE}`;
+        break;
+      default:
+        throw new Error(`Unsupported NoiseCancellationOptions.vendor: ${noiseCancellationOptions.vendor}`);
+    }
+
+    try {
+      log.debug('loading noise cancellation sdk: ', sdkFilePath);
+      const dynamicModule = await dynamicImport(sdkFilePath);
+      log.debug('Loaded noise cancellation sdk:', dynamicModule);
+      const plugin = dynamicModule.default as NoiseCancellationPlugin;
+
+      if (!plugin.isInitialized()) {
+        log.debug('initializing noise cancellation sdk: ', rootDir);
+        await plugin.init({ rootDir });
+        log.debug('noise cancellation sdk initialized!');
+      }
+
+      audioProcessor = {
+        vendor: noiseCancellationOptions.vendor,
+        isInitialized: () => plugin.isInitialized(),
+        isConnected: () => plugin.isConnected(),
+        isEnabled: () => plugin.isEnabled(),
+        disconnect: () => plugin.disconnect(),
+        enable: () => plugin.enable(),
+        disable: () => plugin.disable(),
+        destroy: () => plugin.destroy(),
+        setLogging: (enable: boolean) => plugin.setLogging(enable),
+        connect: (sourceTrack: MediaStreamTrack) => {
+          log.debug('connect: ', sourceTrack.id);
+          if (plugin.isConnected()) {
+            plugin.disconnect();
+          }
+
+          const mediaStream = plugin.connect(new MediaStream([sourceTrack]));
+          if (!mediaStream) {
+            throw new Error('Error connecting with noise cancellation sdk');
+          }
+          const cleanTrack = mediaStream.getAudioTracks()[0];
+          if (!cleanTrack) {
+            throw new Error('Error getting clean track from noise cancellation sdk');
+          }
+          plugin.enable();
+          return cleanTrack;
+        },
+      };
+      audioProcessors.set(noiseCancellationOptions.vendor, audioProcessor);
+
+    } catch (er) {
+      log.error(`Error loading noise cancellation sdk:${sdkFilePath}`, er);
+      throw er;
+    }
+  }
+  return audioProcessor;
+}

package/lib/preflight/preflighttest.ts

@@ -9,7 +9,7 @@ import { getTurnCredentials } from './getturncredentials';
 import { makeStat } from './makestat';
 import { syntheticAudio } from './syntheticaudio';
 import { syntheticVideo } from './syntheticvideo';
-import { waitForSometime } from '../util';
+import { waitForSometime } from '../util/index';
 
 const { WS_SERVER } = require('../util/constants');
 const Log = require('../util/log');

package/lib/room.js

@@ -130,6 +130,7 @@ class Room extends EventEmitter {
     handleLocalParticipantEvents(this, localParticipant);
     handleRecordingEvents(this, signaling.recording);
     handleSignalingEvents(this, signaling);
+    verifyNoiseCancellation(this);
 
     log.info('Created a new Room:', this.name);
     log.debug('Initial RemoteParticipants:', Array.from(this._participants.values()));
@@ -139,6 +140,7 @@ class Room extends EventEmitter {
     return `[Room #${this._instanceId}: ${this.sid}]`;
   }
 
+
   /**
    * Disconnect from the {@link Room}.
    * @returns {this}
@@ -171,6 +173,17 @@ class Room extends EventEmitter {
   }
 }
 
+function verifyNoiseCancellation(room) {
+  const allowedAudioProcessors = room.localParticipant._signaling.audioProcessors;
+  room.localParticipant.audioTracks.forEach(({ track }) => {
+    const noiseCancellation = track.noiseCancellation;
+    if (noiseCancellation && !allowedAudioProcessors.includes(noiseCancellation.vendor)) {
+      room._log.warn(`${noiseCancellation.vendor} is not supported in this room. disabling it permanently`);
+      noiseCancellation.disablePermanently();
+    }
+  });
+}
+
 function rewriteLocalTrackIds(room, trackStats) {
   const localParticipantSignaling = room.localParticipant._signaling;
   return trackStats.reduce((trackStats, trackStat) => {

package/lib/signaling/localparticipant.js

@@ -19,10 +19,11 @@ class LocalParticipantSignaling extends ParticipantSignaling {
    * @param {DataTrackSender|MediaTrackSender} trackSender
    * @param {string} name
    * @param {Track.Priority} priority
+   * @param {?NoiseCancellationVendor} noiseCancellationVendor
    * @returns {LocalTrackPublicationSignaling} publication
    */
-  addTrack(trackSender, name, priority) {
-    const publication = this._createLocalTrackPublicationSignaling(trackSender, name, priority);
+  addTrack(trackSender, name, priority, noiseCancellationVendor = null) {
+    const publication = this._createLocalTrackPublicationSignaling(trackSender, name, priority, noiseCancellationVendor);
     this._trackSendersToPublications.set(trackSender, publication);
     this._publicationsToTrackSenders.set(publication, trackSender);
     super.addTrack(publication);

package/lib/signaling/v2/cancelableroomsignalingpromise.js

@@ -124,9 +124,6 @@ function createCancelableRoomSignalingPromise(token, wsServer, localParticipant,
         reject(new SignalingIncomingMessageInvalidError());
         return;
       }
-
-      const { options: { signaling_region: signalingRegion } } = initialState;
-      localParticipant.setSignalingRegion(signalingRegion);
       resolve(new RoomV2(localParticipant, initialState, transport, peerConnectionManager, options));
     });
 

package/lib/signaling/v2/localparticipant.js

@@ -65,6 +65,10 @@ class LocalParticipantV2 extends LocalParticipantSignaling {
         value: null,
         writable: true
       },
+      audioProcessors: {
+        value: [],
+        writable: true
+      },
       bandwidthProfile: {
         enumerable: true,
         get() {
@@ -125,6 +129,14 @@ class LocalParticipantV2 extends LocalParticipantSignaling {
     }
   }
 
+  /**
+   * Sets the AudioProcessors enabled for this room.
+   * @param {string[]} audioProcessors
+   */
+  setAudioProcessors(audioProcessors) {
+    this.audioProcessors = audioProcessors;
+  }
+
   /**
    * returns current {@link EncodingParametersImpl}.
    * @returns {EncodingParametersImpl}
@@ -184,10 +196,11 @@ class LocalParticipantV2 extends LocalParticipantSignaling {
    * @param {DataTrackSender|MediaTrackSender} trackSender
    * @param {string} name
    * @param {Track.Priority} priority
+   * @param {?NoiseCancellationVendor} noiseCancellationVendor
    * @returns {LocalTrackPublicationV2}
    */
-  _createLocalTrackPublicationSignaling(trackSender, name, priority) {
-    return new this._LocalTrackPublicationV2(trackSender, name, priority, { log: this._log });
+  _createLocalTrackPublicationSignaling(trackSender, name, priority, noiseCancellationVendor) {
+    return new this._LocalTrackPublicationV2(trackSender, name, priority, noiseCancellationVendor, { log: this._log });
   }
 
   /**
@@ -198,8 +211,8 @@ class LocalParticipantV2 extends LocalParticipantSignaling {
    * @param {Track.Priority} priority
    * @returns {this}
    */
-  addTrack(trackSender, name, priority) {
-    super.addTrack(trackSender, name, priority);
+  addTrack(trackSender, name, priority, noiseCancellationVendor) {
+    super.addTrack(trackSender, name, priority, noiseCancellationVendor);
     const publication = this.getPublication(trackSender);
 
     let {

package/lib/signaling/v2/localtrackpublication.js

@@ -13,10 +13,12 @@ class LocalTrackPublicationV2 extends LocalTrackPublicationSignaling {
    * @param {DataTrackSender|MediaTrackSender} trackSender
    * @param {string} name
    * @param {Track.Priority} priority
+   * @param {?NoiseCancellationVendor} noiseCancellationVendor
    * @param {object} [options]
    */
-  constructor(trackSender, name, priority, options) {
+  constructor(trackSender, name, priority, noiseCancellationVendor, options) {
     super(trackSender, name, priority);
+
     Object.defineProperties(this, {
       _log: {
         value: options.log.createLog('default', this)
@@ -24,6 +26,9 @@ class LocalTrackPublicationV2 extends LocalTrackPublicationSignaling {
       _mediaStates: {
         value: { recordings: null },
         writable: true
+      },
+      _noiseCancellationVendor: {
+        value: noiseCancellationVendor,
       }
     });
   }
@@ -33,13 +38,19 @@ class LocalTrackPublicationV2 extends LocalTrackPublicationSignaling {
    * @returns {LocalTrackPublicationV2#Representation} - without the SID
    */
   getState() {
-    return {
+    const state = {
       enabled: this.isEnabled,
       id: this.id,
       kind: this.kind,
       name: this.name,
-      priority: this.updatedPriority
+      priority: this.updatedPriority,
     };
+
+    if (this._noiseCancellationVendor) {
+      // eslint-disable-next-line camelcase
+      state.audio_processor = this._noiseCancellationVendor;
+    }
+    return state;
   }
 
   toString() {

package/lib/signaling/v2/room.js

@@ -52,6 +52,18 @@ class RoomV2 extends RoomSignaling {
     }, options);
 
     localParticipant.setBandwidthProfile(options.bandwidthProfile);
+
+    const { options: { signaling_region: signalingRegion, audio_processors: audioProcessors = [] } } = initialState;
+    localParticipant.setSignalingRegion(signalingRegion);
+
+
+    if (audioProcessors.includes('krisp')) {
+      // Note(mpatwardhan): we add rnnoise as allowed_processor to enable testing our pipeline e2e.
+      audioProcessors.push('rnnoise');
+    }
+
+    localParticipant.setAudioProcessors(audioProcessors);
+
     peerConnectionManager.setIceReconnectTimeout(options.sessionTimeout);
 
     super(localParticipant, initialState.sid, initialState.name, options);
@@ -154,7 +166,7 @@ class RoomV2 extends RoomSignaling {
     this._update(initialState);
 
     // NOTE(mpatwardhan) after initial state we know if publisher_hints are enabled or not
-    // if they are not enabled. we need to undo simulcast that if it was enabled with initial offer.
+    // if they are not enabled. we need to undo simulcast that was enabled with initial offer.
     this._peerConnectionManager.setEffectiveAdaptiveSimulcast(this._publisherHintsSignaling.isSetup);
   }
 

package/package.json

@@ -2,7 +2,7 @@
   "name": "twilio-video",
   "title": "Twilio Video",
   "description": "Twilio Video JavaScript Library",
-  "version": "2.22.2",
+  "version": "2.23.0",
   "homepage": "https://twilio.com",
   "author": "Mark Andrus Roberts <mroberts@twilio.com>",
   "contributors": [
@@ -135,7 +135,7 @@
     "test:framework:install": "npm install chromedriver && npm install selenium-webdriver && npm install geckodriver && npm install puppeteer",
     "test:framework": "npm-run-all test:framework:install test:framework:no-framework test:framework:react",
     "test": "npm-run-all test:unit test:integration",
-    "build:es5": "rimraf ./es5 && tsc tsdef/twilio-video-tests.ts --noEmit --lib es2018,dom && tsc ",
+    "build:es5": "rimraf ./es5 && mkdir -p es5 && cp lib/dynamicImport.js es5/dynamicImport.js && tsc tsdef/twilio-video-tests.ts --noEmit --lib es2018,dom && tsc",
     "build:js": "node ./scripts/build.js ./src/twilio-video.js ./LICENSE.md ./dist/twilio-video.js",
     "build:min.js": "uglifyjs ./dist/twilio-video.js -o ./dist/twilio-video.min.js --comments \"/^! twilio-video.js/\" -b beautify=false,ascii_only=true",
     "build": "npm-run-all clean lint docs test:unit test:integration build:es5 build:js build:min.js test:umd",

package/tsconfig.json

@@ -7,6 +7,7 @@
     "lib": ["es2019", "dom"],
     "downlevelIteration": true,
     "module": "commonjs",
+    "moduleResolution": "node",
     "noImplicitAny": true,
     "noImplicitThis": true,
     "outDir": "./es5",
@@ -16,5 +17,8 @@
   "include": [
     "lib/**/*.ts",
     "lib/**/*.js"
+  ],
+  "exclude": [
+    "lib/dynamicImport.js"
   ]
 }

package/tsdef/AudioProcessor.d.ts

@@ -0,0 +1,65 @@
+import { NoiseCancellationVendor } from './types';
+
+export interface AudioProcessor {
+  /**
+   * @property {NoiseCancellationVendor} vendor
+   */
+  vendor: NoiseCancellationVendor;
+
+  /**
+   * connects and cleans given stream
+   * @returns {MediaStreamTrack} clean track
+   */
+  connect: (sourceTrack: MediaStreamTrack) => MediaStreamTrack
+
+  /**
+   * checks if AudioProcessor is currently enabled
+   * @returns {boolean} true if audio processor is enabled.
+   */
+  isEnabled: () => boolean;
+
+  /**
+   * checks if AudioProcessor is initialized (not destroyed)
+   * @returns {boolean} true if audio processor is initialized.
+   */
+  isInitialized(): boolean;
+
+  /**
+   * checks if AudioProcessor is connected to a stream
+   * @returns {boolean} true if a stream is connected.
+   */
+  isConnected(): boolean;
+
+  /**
+   * enables noise cancellation.
+   * @returns {void}
+   */
+  enable: () => void;
+
+  /**
+   * disables noise cancellation.
+   * @returns {void}
+   */
+  disable: () => void;
+
+  /**
+   * stops processing previously connected stream
+   * @returns {void}
+   */
+  disconnect: () => void;
+
+  /**
+   * destroys the processor freeing up any resources
+   * @returns {void}
+   */
+  destroy:() => void;
+
+  /**
+   * enables/disables logging
+   * @param {boolean} [enable] - Specify true to enable logging
+   * @returns {void}
+   */
+  setLogging:(enable: boolean) => void;
+}
+
+

package/tsdef/LocalAudioTrack.d.ts

@@ -1,5 +1,6 @@
 import { AudioTrack } from './AudioTrack';
 import { LocalTrackOptions } from './LocalTrackOptions';
+import { NoiseCancellation } from './types';
 import { Track } from './Track';
 
 export class LocalAudioTrack extends AudioTrack {
@@ -7,6 +8,7 @@ export class LocalAudioTrack extends AudioTrack {
 
   id: Track.ID;
   isStopped: boolean;
+  noiseCancellation?: NoiseCancellation;
 
   disable(): this;
   enable(enabled?: boolean): this;

package/tsdef/index.d.ts

@@ -1,4 +1,5 @@
-import { CancelablePromise, ConnectOptions, CreateLocalTrackOptions, CreateLocalTracksOptions, LocalTrack } from './types';
+
+import { CancelablePromise, ConnectOptions, CreateLocalAudioTrackOptions, CreateLocalTrackOptions, CreateLocalTracksOptions, LocalTrack } from './types';
 import { PreflightOptions, PreflightTestReport } from './PreflightTypes';
 import { LocalAudioTrack } from './LocalAudioTrack';
 import { LocalVideoTrack } from './LocalVideoTrack';
@@ -11,7 +12,7 @@ export const isSupported: boolean;
 export const version:string;
 export const Logger: Log.RootLogger;
 export function connect(token: string, options?: ConnectOptions): CancelablePromise<Room>;
-export function createLocalAudioTrack(options?: CreateLocalTrackOptions): Promise<LocalAudioTrack>;
+export function createLocalAudioTrack(options?: CreateLocalTracksOptions|CreateLocalAudioTrackOptions): Promise<LocalAudioTrack>;
 export function createLocalTracks(options?: CreateLocalTracksOptions): Promise<LocalTrack[]>;
 export function createLocalVideoTrack(options?: CreateLocalTrackOptions): Promise<LocalVideoTrack>;
 export function runPreflight(token: string, options?: PreflightOptions): PreflightTest;
@@ -55,6 +56,7 @@ export {
   DataTrack,
   DataTrackPublication,
   EncodingParameters,
+  CreateLocalAudioTrackOptions,
   LocalAudioTrackStats,
   LocalTrack,
   LocalTrackPublishOptions,
@@ -72,6 +74,9 @@ export {
   NetworkQualitySendOrRecvStats,
   NetworkQualityStats,
   NetworkQualityVerbosity,
+  NoiseCancellation,
+  NoiseCancellationOptions,
+  NoiseCancellationVendor,
   OpusCodecSettings,
   RemoteAudioTrackStats,
   RemoteTrack,

package/tsdef/types.d.ts

@@ -166,8 +166,29 @@ export interface CreateLocalTrackOptions extends MediaTrackConstraints {
   workaroundWebKitBug1208516?: boolean;
 }
 
+
+export type NoiseCancellationVendor = 'krisp' | 'rnnoise';
+
+export interface NoiseCancellation {
+  readonly vendor: NoiseCancellationVendor;
+  readonly sourceTrack: MediaStreamTrack;
+  readonly isEnabled: boolean;
+
+  enable: () => Promise<void>;
+  disable: () => Promise<void>;
+}
+
+export interface NoiseCancellationOptions {
+  sdkAssetsPath: string;
+  vendor: NoiseCancellationVendor;
+}
+
+export interface CreateLocalAudioTrackOptions extends CreateLocalTrackOptions {
+  noiseCancellationOptions?: NoiseCancellationOptions;
+}
+
 export interface ConnectOptions {
-  audio?: boolean | CreateLocalTrackOptions;
+  audio?: boolean | CreateLocalTrackOptions| CreateLocalAudioTrackOptions;
   automaticSubscription?: boolean;
   bandwidthProfile?: BandwidthProfileOptions;
   dominantSpeaker?: boolean;
@@ -205,7 +226,7 @@ export interface ConnectOptions {
 }
 
 export interface CreateLocalTracksOptions {
-  audio?: boolean | CreateLocalTrackOptions;
+  audio?: boolean | CreateLocalTrackOptions | CreateLocalAudioTrackOptions;
   /**
    * @deprecated
    */