twilio-video 2.26.2 → 2.27.0

package/lib/media/track/videotrack.js

@@ -54,6 +54,10 @@ class VideoTrack extends MediaTrack {
         value: null,
         writable: true,
       },
+      _processorOptions: {
+        value: {},
+        writable: true,
+      },
       _unmuteHandler: {
         value: null,
         writable: true
@@ -152,27 +156,31 @@ class VideoTrack extends MediaTrack {
         const { width = 0, height = 0 } = this.mediaStreamTrack.getSettings();
         // Setting the canvas' dimension triggers a redraw.
         // Only set it if it has changed.
-        if (this._inputFrame.width !== width) {
-          this._inputFrame.width = width;
-          this._inputFrame.height = height;
-
-          if (this._outputFrame) {
-            this._outputFrame.width = width;
-            this._outputFrame.height = height;
+        if (this._outputFrame && this._outputFrame.width !== width) {
+          this._outputFrame.width = width;
+          this._outputFrame.height = height;
+        }
+        if (this._inputFrame) {
+          if (this._inputFrame.width !== width) {
+            this._inputFrame.width = width;
+            this._inputFrame.height = height;
           }
+          this._inputFrame.getContext('2d').drawImage(this._dummyEl, 0, 0, width, height);
         }
-        this._inputFrame.getContext('2d').drawImage(this._dummyEl, 0, 0, width, height);
 
         let result = null;
         try {
-          result = this.processor.processFrame(this._inputFrame, this._outputFrame);
+          const input = this._processorOptions.inputFrameBufferType === 'video' ? this._dummyEl : this._inputFrame;
+          result = this.processor.processFrame(input, this._outputFrame);
         } catch (ex) {
           this._log.debug('Exception detected after calling processFrame.', ex);
         }
         ((result instanceof Promise) ? result : Promise.resolve(result))
           .then(() => {
             if (this._outputFrame) {
-              this.processedTrack.requestFrame();
+              if (typeof this.processedTrack.requestFrame === 'function') {
+                this.processedTrack.requestFrame();
+              }
               this._processorEventObserver.emit('stats');
             }
           })
@@ -216,8 +224,9 @@ class VideoTrack extends MediaTrack {
   _restartProcessor() {
     const processor = this.processor;
     if (processor) {
+      const processorOptions = Object.assign({}, this._processorOptions);
       this.removeProcessor(processor);
-      this.addProcessor(processor);
+      this.addProcessor(processor, processorOptions);
     }
   }
 
@@ -235,8 +244,8 @@ class VideoTrack extends MediaTrack {
 
   /**
    * Add a {@link VideoProcessor} to allow for custom processing of video frames belonging to a VideoTrack.
-   * Only Chrome supports this as of now. Calling this API from a non-supported browser will result in a log warning.
    * @param {VideoProcessor} processor - The {@link VideoProcessor} to use.
+   * @param {AddProcessorOptions} [options] - {@link AddProcessorOptions} to provide.
    * @returns {this}
    * @example
    * class GrayScaleProcessor {
@@ -254,10 +263,7 @@ class VideoTrack extends MediaTrack {
    *   videoTrack.addProcessor(new GrayScaleProcessor(100));
    * });
    */
-  addProcessor(processor) {
-    if (typeof OffscreenCanvas !== 'function') {
-      return this._log.warn('Adding a VideoProcessor is not supported in this browser.');
-    }
+  addProcessor(processor, options) {
     if (!processor || typeof processor.processFrame !== 'function') {
       throw new Error('Received an invalid VideoProcessor from addProcessor.');
     }
@@ -284,13 +290,51 @@ class VideoTrack extends MediaTrack {
       this.mediaStreamTrack.addEventListener('unmute', this._unmuteHandler);
     }
 
+    this._processorOptions = options || {};
+    let { inputFrameBufferType, outputFrameBufferContextType } = this._processorOptions;
+    if (typeof OffscreenCanvas === 'undefined' && inputFrameBufferType === 'offscreencanvas') {
+      throw new Error('OffscreenCanvas is not supported by this browser.');
+    }
+    if (inputFrameBufferType && inputFrameBufferType !== 'video' && inputFrameBufferType !== 'canvas' && inputFrameBufferType !== 'offscreencanvas') {
+      throw new Error(`Invalid inputFrameBufferType of ${inputFrameBufferType}`);
+    }
+    if (!inputFrameBufferType) {
+      inputFrameBufferType = typeof OffscreenCanvas === 'undefined' ? 'canvas' : 'offscreencanvas';
+    }
+
     const { width = 0, height = 0, frameRate = DEFAULT_FRAME_RATE } = this.mediaStreamTrack.getSettings();
-    this._inputFrame = new OffscreenCanvas(width, height);
+    if (inputFrameBufferType === 'offscreencanvas') {
+      this._inputFrame = new OffscreenCanvas(width, height);
+    }
+    if (inputFrameBufferType === 'canvas') {
+      this._inputFrame = document.createElement('canvas');
+    }
+    if (this._inputFrame) {
+      this._inputFrame.width = width;
+      this._inputFrame.height = height;
+    }
+
     this._outputFrame = document.createElement('canvas');
     this._outputFrame.width = width;
     this._outputFrame.height = height;
 
-    this.processedTrack = this._outputFrame.captureStream(0).getTracks()[0];
+    // NOTE(csantos): Initialize the rendering context for future renders. This also ensures
+    // that the correct type is used and on Firefox, it throws an exception if you try to capture
+    // frames prior calling getContext https://bugzilla.mozilla.org/show_bug.cgi?id=1572422
+    outputFrameBufferContextType = outputFrameBufferContextType || '2d';
+    const ctx = this._outputFrame.getContext(outputFrameBufferContextType);
+    if (!ctx) {
+      throw new Error(`Cannot get outputFrameBufferContextType: ${outputFrameBufferContextType}.`);
+    }
+
+    // NOTE(csantos): Zero FPS means we can control when to render the next frame by calling requestFrame.
+    // Some browsers such as Firefox doesn't support requestFrame so we will use default, which is an undefined value.
+    // This means, the browser will use the highest FPS available.
+    const targetFps = typeof CanvasCaptureMediaStreamTrack !== 'undefined' && CanvasCaptureMediaStreamTrack.prototype &&
+      // eslint-disable-next-line
+      typeof CanvasCaptureMediaStreamTrack.prototype.requestFrame === 'function' ? 0 : undefined;
+
+    this.processedTrack = this._outputFrame.captureStream(targetFps).getTracks()[0];
     this.processedTrack.enabled = this.mediaStreamTrack.enabled;
     this.processor = processor;
 
@@ -299,7 +343,9 @@ class VideoTrack extends MediaTrack {
       captureHeight: height,
       captureWidth: width,
       inputFrameRate: frameRate,
-      isRemoteVideoTrack: this.toString().includes('RemoteVideoTrack')
+      isRemoteVideoTrack: this.toString().includes('RemoteVideoTrack'),
+      inputFrameBufferType,
+      outputFrameBufferContextType
     });
     this._updateElementsMediaStreamTrack();
     this._captureFrames();
@@ -440,13 +486,12 @@ class VideoTrack extends MediaTrack {
     this._log.debug('Removing VideoProcessor from the VideoTrack', processor);
     clearTimeout(this._captureTimeoutId);
     this.mediaStreamTrack.removeEventListener('unmute', this._unmuteHandler);
+    this._processorOptions = {};
     this._unmuteHandler = null;
     this._isCapturing = false;
 
     this.processor = null;
     this.processedTrack = null;
-    this._inputFrame.getContext('2d').clearRect(0, 0, this._inputFrame.width, this._inputFrame.height);
-    this._outputFrame.getContext('2d').clearRect(0, 0, this._outputFrame.width, this._outputFrame.height);
     this._inputFrame = null;
     this._outputFrame = null;
 
@@ -484,7 +529,7 @@ function dimensionsChanged(track, elem) {
  * Any exception raised (either synchronously or asynchronously) in `processFrame` will result in the frame being dropped.
  * This callback has the following signature:<br/><br/>
  * <code>processFrame(</code><br/>
- * &nbsp;&nbsp;<code>inputFrameBuffer: OffscreenCanvas,</code><br/>
+ * &nbsp;&nbsp;<code>inputFrameBuffer: OffscreenCanvas | HTMLCanvasElement | HTMLVideoElement,</code><br/>
  * &nbsp;&nbsp;<code>outputFrameBuffer: HTMLCanvasElement</code><br/>
  * <code>): Promise&lt;void&gt; | void;</code>
  *
@@ -501,6 +546,31 @@ function dimensionsChanged(track, elem) {
  * }
  */
 
+/**
+ * Possible options to provide to {@link LocalVideoTrack#addProcessor} and {@link RemoteVideoTrack#addProcessor}.
+ * @typedef {object} AddProcessorOptions
+ * @property {string} [inputFrameBufferType="offscreencanvas"] - This option allows you to specify what kind of input you want to receive in your
+ * Video Processor. The default is `offscreencanvas` and will fallback to a regular `canvas` if the browser does not support it.
+ * Possible values include the following.
+ * <br/>
+ * <br/>
+ * `offscreencanvas` - Your Video Processor will receive an [OffscreenCanvas](https://developer.mozilla.org/en-US/docs/Web/API/OffscreenCanvas)
+ * which is good for canvas-related processing that can be rendered off screen.
+ * <br/>
+ * <br/>
+ * `canvas` - Your Video Processor will receive an [HTMLCanvasElement](https://developer.mozilla.org/en-US/docs/Web/API/HTMLCanvasElement).
+ * This is recommended on browsers that doesn't support `OffscreenCanvas`, or if you need to render the frame on the screen.
+ * <br/>
+ * <br/>
+ * `video` - Your Video Processor will receive an [HTMLVideoElement](https://developer.mozilla.org/en-US/docs/Web/API/HTMLVideoElement).
+ * Use this option if you are processing the frame using WebGL or if you only need to [draw](https://developer.mozilla.org/en-US/docs/Web/API/CanvasRenderingContext2D/drawImage)
+ * the frame directly to your output canvas.
+ * @property {string} [outputFrameBufferContextType="2d"] - The SDK needs the [context type](https://developer.mozilla.org/en-US/docs/Web/API/HTMLCanvasElement/getContext)
+ * that your Video Processor uses in order to properly generate the processed track. For example, if your Video Processor uses WebGL2 (`canvas.getContext('webgl2')`),
+ * you should set `outputFrameBufferContextType` to `webgl2`. Or if you're using Canvas 2D processing (`canvas.getContext('2d')`),
+ * you should set `outputFrameBufferContextType` to `2d`.
+ */
+
 /**
  * The {@link VideoTrack}'s dimensions changed.
  * @param {VideoTrack} track - The {@link VideoTrack} whose dimensions changed

package/package.json

@@ -2,7 +2,7 @@
   "name": "twilio-video",
   "title": "Twilio Video",
   "description": "Twilio Video JavaScript Library",
-  "version": "2.26.2",
+  "version": "2.27.0",
   "homepage": "https://twilio.com",
   "author": "Mark Andrus Roberts <mroberts@twilio.com>",
   "contributors": [

package/tsdef/VideoProcessor.d.ts

@@ -1,3 +1,6 @@
 export class VideoProcessor {
-  processFrame(inputFrameBuffer: OffscreenCanvas, outputFrameBuffer: HTMLCanvasElement): Promise<void> | void;
+  processFrame(
+    inputFrameBuffer: OffscreenCanvas | HTMLCanvasElement | HTMLVideoElement,
+    outputFrameBuffer: HTMLCanvasElement
+  ): Promise<void> | void;
 }

package/tsdef/VideoTrack.d.ts

@@ -1,3 +1,4 @@
+import { AddProcessorOptions } from './types';
 import { MediaTrack } from './MediaTrack';
 import { VideoProcessor } from './VideoProcessor';
 
@@ -14,6 +15,6 @@ export class VideoTrack extends MediaTrack {
   processor: VideoProcessor | null;
   processedTrack: MediaStreamTrack | null;
 
-  addProcessor(processor: VideoProcessor): this;
+  addProcessor(processor: VideoProcessor, options?: AddProcessorOptions): this;
   removeProcessor(processor: VideoProcessor): this;
 }

package/tsdef/index.d.ts

@@ -47,6 +47,7 @@ export { VideoProcessor } from './VideoProcessor';
 export { VideoTrack } from './VideoTrack';
 
 export {
+  AddProcessorOptions,
   AudioCodec,
   AudioCodecSettings,
   AudioLevel,

package/tsdef/types.d.ts

@@ -21,6 +21,11 @@ export type AudioTrackPublication = LocalAudioTrackPublication | RemoteAudioTrac
 export type DataTrackPublication = LocalDataTrackPublication | RemoteDataTrackPublication;
 export type VideoTrackPublication = LocalVideoTrackPublication | RemoteVideoTrackPublication;
 
+export interface AddProcessorOptions {
+  inputFrameBufferType?: 'video' | 'canvas' | 'offscreencanvas';
+  outputFrameBufferContextType?: string;
+}
+
 export interface EncodingParameters {
   maxAudioBitrate?: number | null;
   maxVideoBitrate?: number | null;