shengwang-electron-sdk 4.6.2-rc.1

package/js/Renderer/RendererManager.js

@@ -0,0 +1,250 @@
+"use strict";
+var __importDefault = (this && this.__importDefault) || function (mod) {
+    return (mod && mod.__esModule) ? mod : { "default": mod };
+};
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.RendererManager = void 0;
+const AgoraSdk_1 = __importDefault(require("../AgoraSdk"));
+const AgoraBase_1 = require("../Private/AgoraBase");
+const AgoraMediaBase_1 = require("../Private/AgoraMediaBase");
+const Types_1 = require("../Types");
+const Utils_1 = require("../Utils");
+const IRendererCache_1 = require("./IRendererCache");
+const RendererCache_1 = require("./RendererCache");
+const WebCodecsRenderer_1 = require("./WebCodecsRenderer");
+const WebCodecsRendererCache_1 = require("./WebCodecsRendererCache");
+const WebGLRenderer_1 = require("./WebGLRenderer");
+const YUVCanvasRenderer_1 = require("./YUVCanvasRenderer");
+/**
+ * @ignore
+ */
+class RendererManager {
+    constructor() {
+        /**
+         * Currently, the remote video frame is observed in the pre-renderer position and you can not change it.
+         * the local video frame is observed in the pre-encoder position by default, and you can change it.
+         * @ignore
+         */
+        this.defaultObservedFramePosition = AgoraMediaBase_1.VideoModulePosition.PositionPreRenderer |
+            AgoraMediaBase_1.VideoModulePosition.PositionPreEncoder;
+        this.renderingFps = 15;
+        this._rendererCaches = [];
+        this._context = {
+            renderMode: AgoraMediaBase_1.RenderModeType.RenderModeHidden,
+            mirrorMode: AgoraBase_1.VideoMirrorModeType.VideoMirrorModeDisabled,
+        };
+        this.rendererType = (0, Utils_1.isSupportWebGL)()
+            ? Types_1.RendererType.WEBGL
+            : Types_1.RendererType.SOFTWARE;
+    }
+    setRenderingFps(fps) {
+        this.renderingFps = fps;
+    }
+    set defaultChannelId(channelId) {
+        this._context.channelId = channelId;
+    }
+    get defaultChannelId() {
+        return this._context.channelId ?? '';
+    }
+    get defaultRenderMode() {
+        return this._context.renderMode;
+    }
+    get defaultMirrorMode() {
+        return this._context.mirrorMode;
+    }
+    release() {
+        this.clearRendererCache();
+    }
+    presetRendererContext(context) {
+        //this is for preset default value
+        context.renderMode = context.renderMode || this.defaultRenderMode;
+        context.mirrorMode = context.mirrorMode || this.defaultMirrorMode;
+        context.useWebCodecsDecoder = context.useWebCodecsDecoder || false;
+        context.enableFps = context.enableFps || false;
+        context.position = context.position || this.defaultObservedFramePosition;
+        context.enableAlphaMask = context.enableAlphaMask || false;
+        if (!Utils_1.AgoraEnv.CapabilityManager?.webCodecsDecoderEnabled) {
+            context.useWebCodecsDecoder = false;
+        }
+        switch (context.sourceType) {
+            case AgoraMediaBase_1.VideoSourceType.VideoSourceRemote:
+                if (context.uid === undefined) {
+                    throw new Error('uid is required');
+                }
+                context.channelId = context.channelId ?? this.defaultChannelId;
+                break;
+            case AgoraMediaBase_1.VideoSourceType.VideoSourceMediaPlayer:
+                if (context.mediaPlayerId === undefined) {
+                    throw new Error('mediaPlayerId is required');
+                }
+                context.channelId = '';
+                context.uid = context.mediaPlayerId;
+                break;
+            case undefined:
+                if (context.uid) {
+                    context.sourceType = AgoraMediaBase_1.VideoSourceType.VideoSourceRemote;
+                }
+                break;
+            default:
+                context.channelId = '';
+                context.uid = 0;
+                break;
+        }
+        return context;
+    }
+    addOrRemoveRenderer(context) {
+        // To be compatible with the old API
+        let { setupMode = AgoraBase_1.VideoViewSetupMode.VideoViewSetupAdd } = context;
+        if (!context.view)
+            setupMode = AgoraBase_1.VideoViewSetupMode.VideoViewSetupRemove;
+        switch (setupMode) {
+            case AgoraBase_1.VideoViewSetupMode.VideoViewSetupAdd:
+                return this.addRendererToCache(context);
+            case AgoraBase_1.VideoViewSetupMode.VideoViewSetupRemove:
+                this.removeRendererFromCache(context);
+                return undefined;
+            case AgoraBase_1.VideoViewSetupMode.VideoViewSetupReplace:
+                this.removeRendererFromCache(context);
+                return this.addRendererToCache(context);
+        }
+    }
+    addRendererToCache(context) {
+        const checkedContext = this.presetRendererContext(context);
+        if (!checkedContext.view)
+            return undefined;
+        if (this.findRenderer(checkedContext.view)) {
+            throw new Error('You have already added this view to the renderer');
+        }
+        let rendererCache = this.getRendererCache(checkedContext);
+        if (!rendererCache) {
+            if (context.useWebCodecsDecoder) {
+                rendererCache = new WebCodecsRendererCache_1.WebCodecsRendererCache(checkedContext);
+            }
+            else {
+                rendererCache = new RendererCache_1.RendererCache(checkedContext);
+            }
+            this._rendererCaches.push(rendererCache);
+        }
+        const renderer = this.createRenderer(checkedContext);
+        rendererCache.addRenderer(renderer);
+        return rendererCache;
+    }
+    removeRendererFromCache(context) {
+        const checkedContext = this.presetRendererContext(context);
+        const rendererCache = this.getRendererCache(checkedContext);
+        if (!rendererCache)
+            return;
+        if (checkedContext.view) {
+            const renderer = rendererCache.findRenderer(checkedContext.view);
+            if (!renderer)
+                return;
+            rendererCache.removeRenderer(renderer);
+        }
+        else {
+            rendererCache.removeRenderer();
+        }
+        if (rendererCache.renderers.length === 0) {
+            rendererCache.stopRendering();
+            rendererCache.release();
+            this._rendererCaches.splice(this._rendererCaches.indexOf(rendererCache), 1);
+        }
+    }
+    clearRendererCache() {
+        for (const rendererCache of this._rendererCaches) {
+            rendererCache.stopRendering();
+            rendererCache.release();
+        }
+        this._rendererCaches.splice(0);
+    }
+    getRendererCache(context) {
+        return this._rendererCaches.find((cache) => cache.key === (0, IRendererCache_1.generateRendererCacheKey)(context));
+    }
+    getRendererCachesBySourceType(sourceType) {
+        return this._rendererCaches.filter((cache) => cache.cacheContext.sourceType === sourceType);
+    }
+    getRendererCaches() {
+        return this._rendererCaches;
+    }
+    getRenderers(context) {
+        return this.getRendererCache(context)?.renderers || [];
+    }
+    findRenderer(view) {
+        for (const rendererCache of this._rendererCaches) {
+            const renderer = rendererCache.findRenderer(view);
+            if (renderer)
+                return renderer;
+        }
+        return undefined;
+    }
+    createRenderer(context, rendererType = this.rendererType) {
+        let renderer;
+        switch (rendererType) {
+            case Types_1.RendererType.WEBGL:
+                if (context.useWebCodecsDecoder) {
+                    renderer = new WebCodecsRenderer_1.WebCodecsRenderer();
+                }
+                else {
+                    renderer = new WebGLRenderer_1.WebGLRenderer(this.handleWebGLFallback(context).bind(this));
+                    renderer.bind(context);
+                }
+                break;
+            case Types_1.RendererType.SOFTWARE:
+                renderer = new YUVCanvasRenderer_1.YUVCanvasRenderer();
+                renderer.bind(context);
+                break;
+            default:
+                throw new Error('Unknown renderer type');
+        }
+        renderer.setContext(context);
+        return renderer;
+    }
+    handleWebGLFallback(context) {
+        return (renderer) => {
+            const renderers = this.getRenderers(context);
+            renderer.unbind();
+            const newRenderer = this.createRenderer(context, Types_1.RendererType.SOFTWARE);
+            renderers.splice(renderers.indexOf(renderer), 1, newRenderer);
+        };
+    }
+    handleWebCodecsFallback(context) {
+        let engine = (0, AgoraSdk_1.default)();
+        engine.getMediaEngine().unregisterVideoEncodedFrameObserver({});
+        if (context.uid) {
+            if ((0, IRendererCache_1.isUseConnection)(context)) {
+                engine.setRemoteVideoSubscriptionOptionsEx(context.uid, {
+                    type: AgoraBase_1.VideoStreamType.VideoStreamHigh,
+                    encodedFrameOnly: false,
+                }, {
+                    channelId: context.channelId,
+                    localUid: context.localUid,
+                });
+            }
+            else {
+                engine.setRemoteVideoSubscriptionOptions(context.uid, {
+                    type: AgoraBase_1.VideoStreamType.VideoStreamHigh,
+                    encodedFrameOnly: false,
+                });
+            }
+        }
+        Utils_1.AgoraEnv.enableWebCodecsDecoder = false;
+        Utils_1.AgoraEnv.CapabilityManager?.setWebCodecsDecoderEnabled(false);
+        let renderers = this.getRenderers(context);
+        for (let renderer of renderers) {
+            this.addOrRemoveRenderer({
+                ...renderer.context,
+                setupMode: AgoraBase_1.VideoViewSetupMode.VideoViewSetupReplace,
+            });
+        }
+    }
+    setRendererContext(context) {
+        const checkedContext = this.presetRendererContext(context);
+        for (const rendererCache of this._rendererCaches) {
+            const result = rendererCache.setRendererContext(checkedContext);
+            if (result) {
+                return true;
+            }
+        }
+        return false;
+    }
+}
+exports.RendererManager = RendererManager;

package/js/Renderer/WebCodecsRenderer/index.js

@@ -0,0 +1,109 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.WebCodecsRenderer = void 0;
+const Types_1 = require("../../Types");
+const Utils_1 = require("../../Utils");
+const IRenderer_1 = require("../IRenderer");
+class WebCodecsRenderer extends IRenderer_1.IRenderer {
+    constructor() {
+        super();
+        this.rendererType = Types_1.RendererType.WEBCODECSRENDERER;
+    }
+    bind(context) {
+        super.bind(context);
+        if (!this.canvas)
+            return;
+        this.offscreenCanvas = this.canvas.transferControlToOffscreen();
+        this.gl = (0, Utils_1.getContextByCanvas)(this.offscreenCanvas, this.context.enableAlphaMask);
+        if (!this.gl)
+            return;
+        const vertexShader = this.gl.createShader(this.gl.VERTEX_SHADER);
+        if (!vertexShader)
+            return;
+        this.gl.shaderSource(vertexShader, WebCodecsRenderer.vertexShaderSource);
+        this.gl.compileShader(vertexShader);
+        if (!this.gl.getShaderParameter(vertexShader, this.gl.COMPILE_STATUS)) {
+            throw this.gl.getShaderInfoLog(vertexShader);
+        }
+        const fragmentShader = this.gl.createShader(this.gl.FRAGMENT_SHADER);
+        if (!fragmentShader)
+            return;
+        this.gl.shaderSource(fragmentShader, WebCodecsRenderer.fragmentShaderSource);
+        this.gl.compileShader(fragmentShader);
+        if (!this.gl.getShaderParameter(fragmentShader, this.gl.COMPILE_STATUS)) {
+            throw this.gl.getShaderInfoLog(fragmentShader);
+        }
+        const shaderProgram = this.gl.createProgram();
+        if (!shaderProgram)
+            return;
+        this.gl.attachShader(shaderProgram, vertexShader);
+        this.gl.attachShader(shaderProgram, fragmentShader);
+        this.gl.linkProgram(shaderProgram);
+        if (!this.gl.getProgramParameter(shaderProgram, this.gl.LINK_STATUS)) {
+            throw this.gl.getProgramInfoLog(shaderProgram);
+        }
+        this.gl.useProgram(shaderProgram);
+        // Vertex coordinates, clockwise from bottom-left.
+        const vertexBuffer = this.gl.createBuffer();
+        this.gl.bindBuffer(this.gl.ARRAY_BUFFER, vertexBuffer);
+        this.gl.bufferData(this.gl.ARRAY_BUFFER, new Float32Array([-1.0, -1.0, -1.0, +1.0, +1.0, +1.0, +1.0, -1.0]), this.gl.STATIC_DRAW);
+        const xyLocation = this.gl.getAttribLocation(shaderProgram, 'xy');
+        this.gl.vertexAttribPointer(xyLocation, 2, this.gl.FLOAT, false, 0, 0);
+        this.gl.enableVertexAttribArray(xyLocation);
+        // Create one texture to upload frames to.
+        const texture = this.gl.createTexture();
+        this.gl.bindTexture(this.gl.TEXTURE_2D, texture);
+        this.gl.texParameteri(this.gl.TEXTURE_2D, this.gl.TEXTURE_MAG_FILTER, this.gl.NEAREST);
+        this.gl.texParameteri(this.gl.TEXTURE_2D, this.gl.TEXTURE_MIN_FILTER, this.gl.NEAREST);
+        this.gl.texParameteri(this.gl.TEXTURE_2D, this.gl.TEXTURE_WRAP_S, this.gl.CLAMP_TO_EDGE);
+        this.gl.texParameteri(this.gl.TEXTURE_2D, this.gl.TEXTURE_WRAP_T, this.gl.CLAMP_TO_EDGE);
+    }
+    drawFrame(uid, frame, _codecConfig) {
+        if (!this.offscreenCanvas || !frame)
+            return;
+        this.offscreenCanvas.width = _codecConfig.codedWidth;
+        this.offscreenCanvas.height = _codecConfig.codedHeight;
+        this.updateRenderMode();
+        this.rotateCanvas({
+            rotation: _codecConfig.rotation,
+        });
+        if (!this.gl)
+            return;
+        if (this.gl) {
+            // Upload the frame.
+            this.gl.texImage2D(this.gl.TEXTURE_2D, 0, this.gl.RGBA, this.gl.RGBA, this.gl.UNSIGNED_BYTE, frame);
+            frame.close();
+            // Configure and clear the drawing area.
+            this.gl.viewport(0, 0, this.gl.drawingBufferWidth, this.gl.drawingBufferHeight);
+            this.gl.clearColor(0.0, 0.0, 0.0, 0.0);
+            this.gl.clear(this.gl.COLOR_BUFFER_BIT);
+            // Draw the frame.
+            this.gl.drawArrays(this.gl.TRIANGLE_FAN, 0, 4);
+        }
+        super.drawFrame(uid);
+        this.getFps();
+    }
+    rotateCanvas({ rotation }) {
+        if (!this.offscreenCanvas || !this.canvas)
+            return;
+        this.canvas.style.transform += ` rotateZ(${rotation}deg)`;
+    }
+}
+exports.WebCodecsRenderer = WebCodecsRenderer;
+WebCodecsRenderer.vertexShaderSource = `
+    attribute vec2 xy;
+    varying highp vec2 uv;
+    void main(void) {
+      gl_Position = vec4(xy, 0.0, 1.0);
+      // Map vertex coordinates (-1 to +1) to UV coordinates (0 to 1).
+      // UV coordinates are Y-flipped relative to vertex coordinates.
+      uv = vec2((1.0 + xy.x) / 2.0, (1.0 - xy.y) / 2.0);
+    }
+  `;
+WebCodecsRenderer.fragmentShaderSource = `
+    varying highp vec2 uv;
+    uniform sampler2D texture;
+    void main(void) {
+      gl_FragColor = texture2D(texture, uv);
+    }
+  `;

package/js/Renderer/WebCodecsRendererCache.js

@@ -0,0 +1,127 @@
+"use strict";
+var __importDefault = (this && this.__importDefault) || function (mod) {
+    return (mod && mod.__esModule) ? mod : { "default": mod };
+};
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.WebCodecsRendererCache = void 0;
+const AgoraSdk_1 = __importDefault(require("../AgoraSdk"));
+const index_1 = require("../Decoder/index");
+const AgoraBase_1 = require("../Private/AgoraBase");
+const IrisApiEngine_1 = require("../Private/internal/IrisApiEngine");
+const Types_1 = require("../Types");
+const Utils_1 = require("../Utils");
+const IRendererCache_1 = require("./IRendererCache");
+class WebCodecsRendererCache extends IRendererCache_1.IRendererCache {
+    constructor(context) {
+        super(context);
+        this._firstFrame = true;
+        this._engine = (0, AgoraSdk_1.default)();
+        this._decoder = new index_1.WebCodecsDecoder(this.renderers, this.onDecoderError.bind(this), context);
+        this.draw();
+    }
+    onDecoderError(e) {
+        (0, Utils_1.logInfo)('webCodecsDecoder decode failed, fallback to native decoder', e);
+        Utils_1.AgoraEnv.AgoraRendererManager?.handleWebCodecsFallback(this.cacheContext);
+    }
+    onEncodedVideoFrameReceived(...[data, buffer]) {
+        let _data;
+        try {
+            _data = JSON.parse(data) ?? {};
+        }
+        catch (e) {
+            _data = {};
+        }
+        if (Object.keys(_data).length === 0 ||
+            !this._decoder ||
+            this.cacheContext.uid !== _data.uid)
+            return;
+        if (this._firstFrame) {
+            for (let renderer of this.renderers) {
+                if (renderer.rendererType !== Types_1.RendererType.WEBCODECSRENDERER) {
+                    continue;
+                }
+                renderer.bind(renderer.context);
+            }
+            try {
+                this._decoder.decoderConfigure(_data.videoEncodedFrameInfo);
+            }
+            catch (error) {
+                (0, Utils_1.logInfo)(error);
+                return;
+            }
+            this._firstFrame = false;
+        }
+        if (this.shouldFallback(_data.videoEncodedFrameInfo)) {
+            Utils_1.AgoraEnv.AgoraRendererManager?.handleWebCodecsFallback(this.cacheContext);
+        }
+        else {
+            this._decoder.decodeFrame(buffer, _data.videoEncodedFrameInfo, new Date().getTime());
+        }
+    }
+    draw() {
+        if ((0, IRendererCache_1.isUseConnection)(this.cacheContext)) {
+            this._engine?.setRemoteVideoSubscriptionOptionsEx(this.cacheContext.uid, {
+                type: AgoraBase_1.VideoStreamType.VideoStreamHigh,
+                encodedFrameOnly: true,
+            }, {
+                channelId: this.cacheContext.channelId,
+                localUid: this.cacheContext.localUid,
+            });
+        }
+        else {
+            this._engine?.setRemoteVideoSubscriptionOptions(this.cacheContext.uid, {
+                type: AgoraBase_1.VideoStreamType.VideoStreamHigh,
+                encodedFrameOnly: true,
+            });
+        }
+        IrisApiEngine_1.AgoraElectronBridge.OnEvent(`call_back_with_encoded_video_frame_${this.cacheContext.uid}`, (...params) => {
+            try {
+                this.onEncodedVideoFrameReceived(...params);
+            }
+            catch (e) {
+                console.error(e);
+            }
+        });
+    }
+    shouldFallback(frameInfo) {
+        let shouldFallback = false;
+        if (!frameInfo.codecType) {
+            shouldFallback = true;
+            (0, Utils_1.logInfo)('codecType is not supported, fallback to native decoder');
+        }
+        else {
+            const mapping = Utils_1.AgoraEnv.CapabilityManager?.frameCodecMapping[frameInfo.codecType];
+            if (mapping === undefined) {
+                shouldFallback = true;
+                (0, Utils_1.logInfo)('codecType is not supported, fallback to native decoder');
+            }
+            else if (mapping.minWidth >= frameInfo.width &&
+                mapping.minHeight >= frameInfo.height &&
+                mapping.maxWidth <= frameInfo.width &&
+                mapping.maxHeight <= frameInfo.height) {
+                shouldFallback = true;
+                (0, Utils_1.logInfo)('frame size is not supported, fallback to native decoder');
+            }
+        }
+        return shouldFallback;
+    }
+    release() {
+        IrisApiEngine_1.AgoraElectronBridge.UnEvent(`call_back_with_encoded_video_frame_${this.cacheContext.uid}`);
+        this._decoder?.release();
+        this._decoder = null;
+        super.release();
+    }
+    fetchVideoFrame() {
+        return { hasMoreFrame: false, needRender: false };
+    }
+    renderFrame() {
+        return;
+    }
+    startRendering() {
+        return;
+    }
+    stopRendering() {
+        return;
+    }
+}
+exports.WebCodecsRendererCache = WebCodecsRendererCache;