twilio-video 2.26.2 → 2.27.0

package/CHANGELOG.md

@@ -2,6 +2,43 @@ The Twilio Programmable Video SDKs use [Semantic Versioning](http://www.semver.o
 
 **Version 1.x reached End of Life on September 8th, 2021.** See the changelog entry [here](https://www.twilio.com/changelog/end-of-life-complete-for-unsupported-versions-of-the-programmable-video-sdk). Support for the 1.x version ended on December 4th, 2020.
 
+2.27.0 (March 21, 2023)
+=======================
+
+Changes
+-------
+
+`VideoTrack.addProcessor` now works on browsers that support `OffscreenCanvas` as well as `HTMLCanvasElement`. When used with
+[@twilio/video-processors v2.0.0](https://github.com/twilio/twilio-video-processors.js/blob/2.0.0/CHANGELOG.md), the Virtual
+Background feature will work on browsers that supports [WebGL2](https://developer.mozilla.org/en-US/docs/Web/API/WebGL2RenderingContext).
+See [VideoTrack.addProcessor](https://sdk.twilio.com/js/video/releases/2.27.0/docs/VideoTrack.html#addProcessor__anchor) and
+[@twilio/video-processors v2.0.0](https://github.com/twilio/twilio-video-processors.js/blob/2.0.0/CHANGELOG.md) for details.
+
+### Example
+
+```ts
+import { createLocalVideoTrack } from 'twilio-video';
+import { Pipeline, VirtualBackgroundProcessor } from '@twilio/video-processors';
+
+const virtualBackgroundProcessor = new VirtualBackgroundProcessor({
+  pipeline: Pipeline.WebGL2,
+  // ...otherOptions
+});
+
+await virtualBackgroundProcessor.loadModel();
+
+const videoTrack = await createLocalVideoTrack({
+  width: 640,
+  height: 480,
+  frameRate: 24
+});
+
+videoTrack.addProcessor(processor, {
+  inputFrameBufferType: 'video',
+  outputFrameBufferContextType: 'webgl2',
+});
+```
+
 2.26.2 (February 21, 2023)
 ==========================
 

package/README.md

@@ -74,7 +74,7 @@ Releases of twilio-video.js are hosted on a CDN, and you can include these
 directly in your web app using a <script> tag.
 
 ```html
-<script src="//sdk.twilio.com/js/video/releases/2.26.2/twilio-video.min.js"></script>
+<script src="//sdk.twilio.com/js/video/releases/2.27.0/twilio-video.min.js"></script>
 ```
 
 Using this method, twilio-video.js will set a browser global:

package/dist/twilio-video.js

@@ -1,4 +1,4 @@
-/*! twilio-video.js 2.26.2
+/*! twilio-video.js 2.27.0
 
 The following license applies to all parts of this software except as
 documented below.
@@ -3873,8 +3873,10 @@ function restartWhenInadvertentlyStopped(localMediaTrack) {
         }).catch(function (error) {
             log.warn('Failed to detect silence:', error);
         }).finally(function () {
-            // Pause the dummy element again.
-            el.pause();
+            // Pause the dummy element again, if there is no processed track.
+            if (!localMediaTrack.processedTrack) {
+                el.pause();
+            }
         });
     }
     function shouldReacquireTrack() {
@@ -4232,8 +4234,8 @@ var LocalVideoTrack = /** @class */ (function (_super) {
     };
     /**
      * Add a {@link VideoProcessor} to allow for custom processing of video frames belonging to a VideoTrack.
-     * Only Chrome supports this as of now. Calling this API from a non-supported browser will result in a log warning.
      * @param {VideoProcessor} processor - The {@link VideoProcessor} to use.
+     * @param {AddProcessorOptions} [options] - {@link AddProcessorOptions} to provide.
      * @returns {this}
      * @example
      * class GrayScaleProcessor {
@@ -4433,9 +4435,10 @@ function workaroundSilentLocalVideo(localVideoTrack, doc) {
         }).catch(function (error) {
             log.warn('Failed to detect silence and restart:', error);
         }).finally(function () {
-            // If silent frames were not detected, then pause the dummy element again.
+            // If silent frames were not detected, then pause the dummy element again,
+            // if there is no processed track.
             el = localVideoTrack._dummyEl;
-            if (el && !el.paused) {
+            if (el && !el.paused && !localVideoTrack.processedTrack) {
                 el.pause();
             }
             // Reset the unmute handler.
@@ -6351,8 +6354,8 @@ var RemoteVideoTrack = /** @class */ (function (_super) {
      * Add a {@link VideoProcessor} to allow for custom processing of video frames belonging to a VideoTrack.
      * When a Participant un-publishes and re-publishes a VideoTrack, a new RemoteVideoTrack is created and
      * any VideoProcessors attached to the previous RemoteVideoTrack would have to be re-added again.
-     * Only Chrome supports this as of now. Calling this API from a non-supported browser will result in a log warning.
      * @param {VideoProcessor} processor - The {@link VideoProcessor} to use.
+     * @param {AddProcessorOptions} [options] - {@link AddProcessorOptions} to provide.
      * @returns {this}
      * @example
      * class GrayScaleProcessor {
@@ -7019,10 +7022,10 @@ var VideoProcessorEventObserver = /** @class */ (function (_super) {
         if (!this._processorInfo) {
             return {};
         }
-        var _a = this._processorInfo, processor = _a.processor, captureHeight = _a.captureHeight, captureWidth = _a.captureWidth, inputFrameRate = _a.inputFrameRate, isRemoteVideoTrack = _a.isRemoteVideoTrack;
-        var data = { captureHeight: captureHeight, captureWidth: captureWidth, inputFrameRate: inputFrameRate, isRemoteVideoTrack: isRemoteVideoTrack };
+        var _a = this._processorInfo, processor = _a.processor, captureHeight = _a.captureHeight, captureWidth = _a.captureWidth, inputFrameRate = _a.inputFrameRate, isRemoteVideoTrack = _a.isRemoteVideoTrack, inputFrameBufferType = _a.inputFrameBufferType, outputFrameBufferContextType = _a.outputFrameBufferContextType;
+        var data = { captureHeight: captureHeight, captureWidth: captureWidth, inputFrameRate: inputFrameRate, isRemoteVideoTrack: isRemoteVideoTrack, inputFrameBufferType: inputFrameBufferType, outputFrameBufferContextType: outputFrameBufferContextType };
         data.name = processor._name || 'VideoProcessor';
-        ['assetsPath', 'blurFilterRadius', 'fitType', 'isSimdEnabled', 'maskBlurRadius', 'version'].forEach(function (prop) {
+        ['assetsPath', 'blurFilterRadius', 'debounce', 'fitType', 'isSimdEnabled', 'maskBlurRadius', 'pipeline', 'version'].forEach(function (prop) {
             var val = processor["_" + prop];
             if (typeof val !== 'undefined') {
                 data[prop] = val;
@@ -7160,6 +7163,10 @@ var VideoTrack = /** @class */ (function (_super) {
                 value: null,
                 writable: true,
             },
+            _processorOptions: {
+                value: {},
+                writable: true,
+            },
             _unmuteHandler: {
                 value: null,
                 writable: true
@@ -7251,18 +7258,21 @@ var VideoTrack = /** @class */ (function (_super) {
                 var _a = _this.mediaStreamTrack.getSettings(), _b = _a.width, width = _b === void 0 ? 0 : _b, _c = _a.height, height = _c === void 0 ? 0 : _c;
                 // Setting the canvas' dimension triggers a redraw.
                 // Only set it if it has changed.
-                if (_this._inputFrame.width !== width) {
-                    _this._inputFrame.width = width;
-                    _this._inputFrame.height = height;
-                    if (_this._outputFrame) {
-                        _this._outputFrame.width = width;
-                        _this._outputFrame.height = height;
+                if (_this._outputFrame && _this._outputFrame.width !== width) {
+                    _this._outputFrame.width = width;
+                    _this._outputFrame.height = height;
+                }
+                if (_this._inputFrame) {
+                    if (_this._inputFrame.width !== width) {
+                        _this._inputFrame.width = width;
+                        _this._inputFrame.height = height;
                     }
+                    _this._inputFrame.getContext('2d').drawImage(_this._dummyEl, 0, 0, width, height);
                 }
-                _this._inputFrame.getContext('2d').drawImage(_this._dummyEl, 0, 0, width, height);
                 var result = null;
                 try {
-                    result = _this.processor.processFrame(_this._inputFrame, _this._outputFrame);
+                    var input = _this._processorOptions.inputFrameBufferType === 'video' ? _this._dummyEl : _this._inputFrame;
+                    result = _this.processor.processFrame(input, _this._outputFrame);
                 }
                 catch (ex) {
                     _this._log.debug('Exception detected after calling processFrame.', ex);
@@ -7270,7 +7280,9 @@ var VideoTrack = /** @class */ (function (_super) {
                 ((result instanceof Promise) ? result : Promise.resolve(result))
                     .then(function () {
                     if (_this._outputFrame) {
-                        _this.processedTrack.requestFrame();
+                        if (typeof _this.processedTrack.requestFrame === 'function') {
+                            _this.processedTrack.requestFrame();
+                        }
                         _this._processorEventObserver.emit('stats');
                     }
                 })
@@ -7313,8 +7325,9 @@ var VideoTrack = /** @class */ (function (_super) {
     VideoTrack.prototype._restartProcessor = function () {
         var processor = this.processor;
         if (processor) {
+            var processorOptions = Object.assign({}, this._processorOptions);
             this.removeProcessor(processor);
-            this.addProcessor(processor);
+            this.addProcessor(processor, processorOptions);
         }
     };
     /**
@@ -7329,8 +7342,8 @@ var VideoTrack = /** @class */ (function (_super) {
     };
     /**
      * Add a {@link VideoProcessor} to allow for custom processing of video frames belonging to a VideoTrack.
-     * Only Chrome supports this as of now. Calling this API from a non-supported browser will result in a log warning.
      * @param {VideoProcessor} processor - The {@link VideoProcessor} to use.
+     * @param {AddProcessorOptions} [options] - {@link AddProcessorOptions} to provide.
      * @returns {this}
      * @example
      * class GrayScaleProcessor {
@@ -7348,11 +7361,8 @@ var VideoTrack = /** @class */ (function (_super) {
      *   videoTrack.addProcessor(new GrayScaleProcessor(100));
      * });
      */
-    VideoTrack.prototype.addProcessor = function (processor) {
+    VideoTrack.prototype.addProcessor = function (processor, options) {
         var _this = this;
-        if (typeof OffscreenCanvas !== 'function') {
-            return this._log.warn('Adding a VideoProcessor is not supported in this browser.');
-        }
         if (!processor || typeof processor.processFrame !== 'function') {
             throw new Error('Received an invalid VideoProcessor from addProcessor.');
         }
@@ -7376,12 +7386,46 @@ var VideoTrack = /** @class */ (function (_super) {
             };
             this.mediaStreamTrack.addEventListener('unmute', this._unmuteHandler);
         }
-        var _a = this.mediaStreamTrack.getSettings(), _b = _a.width, width = _b === void 0 ? 0 : _b, _c = _a.height, height = _c === void 0 ? 0 : _c, _d = _a.frameRate, frameRate = _d === void 0 ? DEFAULT_FRAME_RATE : _d;
-        this._inputFrame = new OffscreenCanvas(width, height);
+        this._processorOptions = options || {};
+        var _a = this._processorOptions, inputFrameBufferType = _a.inputFrameBufferType, outputFrameBufferContextType = _a.outputFrameBufferContextType;
+        if (typeof OffscreenCanvas === 'undefined' && inputFrameBufferType === 'offscreencanvas') {
+            throw new Error('OffscreenCanvas is not supported by this browser.');
+        }
+        if (inputFrameBufferType && inputFrameBufferType !== 'video' && inputFrameBufferType !== 'canvas' && inputFrameBufferType !== 'offscreencanvas') {
+            throw new Error("Invalid inputFrameBufferType of " + inputFrameBufferType);
+        }
+        if (!inputFrameBufferType) {
+            inputFrameBufferType = typeof OffscreenCanvas === 'undefined' ? 'canvas' : 'offscreencanvas';
+        }
+        var _b = this.mediaStreamTrack.getSettings(), _c = _b.width, width = _c === void 0 ? 0 : _c, _d = _b.height, height = _d === void 0 ? 0 : _d, _e = _b.frameRate, frameRate = _e === void 0 ? DEFAULT_FRAME_RATE : _e;
+        if (inputFrameBufferType === 'offscreencanvas') {
+            this._inputFrame = new OffscreenCanvas(width, height);
+        }
+        if (inputFrameBufferType === 'canvas') {
+            this._inputFrame = document.createElement('canvas');
+        }
+        if (this._inputFrame) {
+            this._inputFrame.width = width;
+            this._inputFrame.height = height;
+        }
         this._outputFrame = document.createElement('canvas');
         this._outputFrame.width = width;
         this._outputFrame.height = height;
-        this.processedTrack = this._outputFrame.captureStream(0).getTracks()[0];
+        // NOTE(csantos): Initialize the rendering context for future renders. This also ensures
+        // that the correct type is used and on Firefox, it throws an exception if you try to capture
+        // frames prior calling getContext https://bugzilla.mozilla.org/show_bug.cgi?id=1572422
+        outputFrameBufferContextType = outputFrameBufferContextType || '2d';
+        var ctx = this._outputFrame.getContext(outputFrameBufferContextType);
+        if (!ctx) {
+            throw new Error("Cannot get outputFrameBufferContextType: " + outputFrameBufferContextType + ".");
+        }
+        // NOTE(csantos): Zero FPS means we can control when to render the next frame by calling requestFrame.
+        // Some browsers such as Firefox doesn't support requestFrame so we will use default, which is an undefined value.
+        // This means, the browser will use the highest FPS available.
+        var targetFps = typeof CanvasCaptureMediaStreamTrack !== 'undefined' && CanvasCaptureMediaStreamTrack.prototype &&
+            // eslint-disable-next-line
+            typeof CanvasCaptureMediaStreamTrack.prototype.requestFrame === 'function' ? 0 : undefined;
+        this.processedTrack = this._outputFrame.captureStream(targetFps).getTracks()[0];
         this.processedTrack.enabled = this.mediaStreamTrack.enabled;
         this.processor = processor;
         this._processorEventObserver.emit('add', {
@@ -7389,7 +7433,9 @@ var VideoTrack = /** @class */ (function (_super) {
             captureHeight: height,
             captureWidth: width,
             inputFrameRate: frameRate,
-            isRemoteVideoTrack: this.toString().includes('RemoteVideoTrack')
+            isRemoteVideoTrack: this.toString().includes('RemoteVideoTrack'),
+            inputFrameBufferType: inputFrameBufferType,
+            outputFrameBufferContextType: outputFrameBufferContextType
         });
         this._updateElementsMediaStreamTrack();
         this._captureFrames();
@@ -7526,12 +7572,11 @@ var VideoTrack = /** @class */ (function (_super) {
         this._log.debug('Removing VideoProcessor from the VideoTrack', processor);
         clearTimeout(this._captureTimeoutId);
         this.mediaStreamTrack.removeEventListener('unmute', this._unmuteHandler);
+        this._processorOptions = {};
         this._unmuteHandler = null;
         this._isCapturing = false;
         this.processor = null;
         this.processedTrack = null;
-        this._inputFrame.getContext('2d').clearRect(0, 0, this._inputFrame.width, this._inputFrame.height);
-        this._outputFrame.getContext('2d').clearRect(0, 0, this._outputFrame.width, this._outputFrame.height);
         this._inputFrame = null;
         this._outputFrame = null;
         this._updateElementsMediaStreamTrack();
@@ -7565,7 +7610,7 @@ function dimensionsChanged(track, elem) {
  * Any exception raised (either synchronously or asynchronously) in `processFrame` will result in the frame being dropped.
  * This callback has the following signature:<br/><br/>
  * <code>processFrame(</code><br/>
- * &nbsp;&nbsp;<code>inputFrameBuffer: OffscreenCanvas,</code><br/>
+ * &nbsp;&nbsp;<code>inputFrameBuffer: OffscreenCanvas | HTMLCanvasElement | HTMLVideoElement,</code><br/>
  * &nbsp;&nbsp;<code>outputFrameBuffer: HTMLCanvasElement</code><br/>
  * <code>): Promise&lt;void&gt; | void;</code>
  *
@@ -7581,6 +7626,30 @@ function dimensionsChanged(track, elem) {
  *   }
  * }
  */
+/**
+ * Possible options to provide to {@link LocalVideoTrack#addProcessor} and {@link RemoteVideoTrack#addProcessor}.
+ * @typedef {object} AddProcessorOptions
+ * @property {string} [inputFrameBufferType="offscreencanvas"] - This option allows you to specify what kind of input you want to receive in your
+ * Video Processor. The default is `offscreencanvas` and will fallback to a regular `canvas` if the browser does not support it.
+ * Possible values include the following.
+ * <br/>
+ * <br/>
+ * `offscreencanvas` - Your Video Processor will receive an [OffscreenCanvas](https://developer.mozilla.org/en-US/docs/Web/API/OffscreenCanvas)
+ * which is good for canvas-related processing that can be rendered off screen.
+ * <br/>
+ * <br/>
+ * `canvas` - Your Video Processor will receive an [HTMLCanvasElement](https://developer.mozilla.org/en-US/docs/Web/API/HTMLCanvasElement).
+ * This is recommended on browsers that doesn't support `OffscreenCanvas`, or if you need to render the frame on the screen.
+ * <br/>
+ * <br/>
+ * `video` - Your Video Processor will receive an [HTMLVideoElement](https://developer.mozilla.org/en-US/docs/Web/API/HTMLVideoElement).
+ * Use this option if you are processing the frame using WebGL or if you only need to [draw](https://developer.mozilla.org/en-US/docs/Web/API/CanvasRenderingContext2D/drawImage)
+ * the frame directly to your output canvas.
+ * @property {string} [outputFrameBufferContextType="2d"] - The SDK needs the [context type](https://developer.mozilla.org/en-US/docs/Web/API/HTMLCanvasElement/getContext)
+ * that your Video Processor uses in order to properly generate the processed track. For example, if your Video Processor uses WebGL2 (`canvas.getContext('webgl2')`),
+ * you should set `outputFrameBufferContextType` to `webgl2`. Or if you're using Canvas 2D processing (`canvas.getContext('2d')`),
+ * you should set `outputFrameBufferContextType` to `2d`.
+ */
 /**
  * The {@link VideoTrack}'s dimensions changed.
  * @param {VideoTrack} track - The {@link VideoTrack} whose dimensions changed
@@ -30083,7 +30152,7 @@ module.exports={
   "name": "twilio-video",
   "title": "Twilio Video",
   "description": "Twilio Video JavaScript Library",
-  "version": "2.26.2",
+  "version": "2.27.0",
   "homepage": "https://twilio.com",
   "author": "Mark Andrus Roberts <mroberts@twilio.com>",
   "contributors": [