shengwang-electron-sdk 4.6.2-rc.1

package/ts/Renderer/WebCodecsRenderer/index.ts

@@ -0,0 +1,148 @@
+import { VideoFrame } from '../../Private/AgoraMediaBase';
+import { CodecConfigInfo, RendererContext, RendererType } from '../../Types';
+import { getContextByCanvas } from '../../Utils';
+import { IRenderer } from '../IRenderer';
+
+export class WebCodecsRenderer extends IRenderer {
+  gl?: WebGLRenderingContext | WebGL2RenderingContext | null;
+  // eslint-disable-next-line auto-import/auto-import
+  offscreenCanvas: OffscreenCanvas | undefined;
+
+  constructor() {
+    super();
+    this.rendererType = RendererType.WEBCODECSRENDERER;
+  }
+
+  static vertexShaderSource = `
+    attribute vec2 xy;
+    varying highp vec2 uv;
+    void main(void) {
+      gl_Position = vec4(xy, 0.0, 1.0);
+      // Map vertex coordinates (-1 to +1) to UV coordinates (0 to 1).
+      // UV coordinates are Y-flipped relative to vertex coordinates.
+      uv = vec2((1.0 + xy.x) / 2.0, (1.0 - xy.y) / 2.0);
+    }
+  `;
+  static fragmentShaderSource = `
+    varying highp vec2 uv;
+    uniform sampler2D texture;
+    void main(void) {
+      gl_FragColor = texture2D(texture, uv);
+    }
+  `;
+
+  bind(context: RendererContext) {
+    super.bind(context);
+    if (!this.canvas) return;
+    this.offscreenCanvas = this.canvas.transferControlToOffscreen();
+    this.gl = getContextByCanvas(
+      this.offscreenCanvas,
+      this.context.enableAlphaMask
+    );
+    if (!this.gl) return;
+    const vertexShader = this.gl.createShader(this.gl.VERTEX_SHADER);
+    if (!vertexShader) return;
+    this.gl.shaderSource(vertexShader, WebCodecsRenderer.vertexShaderSource);
+    this.gl.compileShader(vertexShader);
+    if (!this.gl.getShaderParameter(vertexShader, this.gl.COMPILE_STATUS)) {
+      throw this.gl.getShaderInfoLog(vertexShader);
+    }
+    const fragmentShader = this.gl.createShader(this.gl.FRAGMENT_SHADER);
+    if (!fragmentShader) return;
+    this.gl.shaderSource(
+      fragmentShader,
+      WebCodecsRenderer.fragmentShaderSource
+    );
+    this.gl.compileShader(fragmentShader);
+    if (!this.gl.getShaderParameter(fragmentShader, this.gl.COMPILE_STATUS)) {
+      throw this.gl.getShaderInfoLog(fragmentShader);
+    }
+    const shaderProgram = this.gl.createProgram();
+    if (!shaderProgram) return;
+    this.gl.attachShader(shaderProgram, vertexShader);
+    this.gl.attachShader(shaderProgram, fragmentShader);
+    this.gl.linkProgram(shaderProgram);
+    if (!this.gl.getProgramParameter(shaderProgram, this.gl.LINK_STATUS)) {
+      throw this.gl.getProgramInfoLog(shaderProgram);
+    }
+    this.gl.useProgram(shaderProgram);
+    // Vertex coordinates, clockwise from bottom-left.
+    const vertexBuffer = this.gl.createBuffer();
+    this.gl.bindBuffer(this.gl.ARRAY_BUFFER, vertexBuffer);
+    this.gl.bufferData(
+      this.gl.ARRAY_BUFFER,
+      new Float32Array([-1.0, -1.0, -1.0, +1.0, +1.0, +1.0, +1.0, -1.0]),
+      this.gl.STATIC_DRAW
+    );
+    const xyLocation = this.gl.getAttribLocation(shaderProgram, 'xy');
+    this.gl.vertexAttribPointer(xyLocation, 2, this.gl.FLOAT, false, 0, 0);
+    this.gl.enableVertexAttribArray(xyLocation);
+    // Create one texture to upload frames to.
+    const texture = this.gl.createTexture();
+    this.gl.bindTexture(this.gl.TEXTURE_2D, texture);
+    this.gl.texParameteri(
+      this.gl.TEXTURE_2D,
+      this.gl.TEXTURE_MAG_FILTER,
+      this.gl.NEAREST
+    );
+    this.gl.texParameteri(
+      this.gl.TEXTURE_2D,
+      this.gl.TEXTURE_MIN_FILTER,
+      this.gl.NEAREST
+    );
+    this.gl.texParameteri(
+      this.gl.TEXTURE_2D,
+      this.gl.TEXTURE_WRAP_S,
+      this.gl.CLAMP_TO_EDGE
+    );
+    this.gl.texParameteri(
+      this.gl.TEXTURE_2D,
+      this.gl.TEXTURE_WRAP_T,
+      this.gl.CLAMP_TO_EDGE
+    );
+  }
+
+  drawFrame(uid: number, frame: any, _codecConfig: CodecConfigInfo) {
+    if (!this.offscreenCanvas || !frame) return;
+
+    this.offscreenCanvas.width = _codecConfig.codedWidth!;
+    this.offscreenCanvas.height = _codecConfig.codedHeight!;
+
+    this.updateRenderMode();
+    this.rotateCanvas({
+      rotation: _codecConfig.rotation,
+    });
+    if (!this.gl) return;
+
+    if (this.gl) {
+      // Upload the frame.
+      this.gl.texImage2D(
+        this.gl.TEXTURE_2D,
+        0,
+        this.gl.RGBA,
+        this.gl.RGBA,
+        this.gl.UNSIGNED_BYTE,
+        frame
+      );
+      frame.close();
+      // Configure and clear the drawing area.
+      this.gl.viewport(
+        0,
+        0,
+        this.gl.drawingBufferWidth,
+        this.gl.drawingBufferHeight
+      );
+      this.gl.clearColor(0.0, 0.0, 0.0, 0.0);
+      this.gl.clear(this.gl.COLOR_BUFFER_BIT);
+      // Draw the frame.
+      this.gl.drawArrays(this.gl.TRIANGLE_FAN, 0, 4);
+    }
+    super.drawFrame(uid);
+    this.getFps();
+  }
+
+  protected override rotateCanvas({ rotation }: VideoFrame) {
+    if (!this.offscreenCanvas || !this.canvas) return;
+    this.canvas.style.transform += ` rotateZ(${rotation}deg)`;
+  }
+}

package/ts/Renderer/WebCodecsRendererCache.ts

@@ -0,0 +1,153 @@
+import createAgoraRtcEngine from '../AgoraSdk';
+import { WebCodecsDecoder } from '../Decoder/index';
+import { EncodedVideoFrameInfo, VideoStreamType } from '../Private/AgoraBase';
+import { IRtcEngineEx } from '../Private/IAgoraRtcEngineEx';
+import { AgoraElectronBridge } from '../Private/internal/IrisApiEngine';
+
+import { RendererContext, RendererType } from '../Types';
+import { AgoraEnv, logInfo } from '../Utils';
+
+import { IRendererCache, isUseConnection } from './IRendererCache';
+import { WebCodecsRenderer } from './WebCodecsRenderer/index';
+
+export class WebCodecsRendererCache extends IRendererCache {
+  private _decoder?: WebCodecsDecoder | null;
+  private _engine?: IRtcEngineEx;
+  private _firstFrame = true;
+
+  constructor(context: RendererContext) {
+    super(context);
+    this._engine = createAgoraRtcEngine();
+    this._decoder = new WebCodecsDecoder(
+      this.renderers as WebCodecsRenderer[],
+      this.onDecoderError.bind(this),
+      context
+    );
+    this.draw();
+  }
+
+  onDecoderError(e: any) {
+    logInfo('webCodecsDecoder decode failed, fallback to native decoder', e);
+    AgoraEnv.AgoraRendererManager?.handleWebCodecsFallback(this.cacheContext);
+  }
+
+  onEncodedVideoFrameReceived(...[data, buffer]: any) {
+    let _data: any;
+    try {
+      _data = JSON.parse(data) ?? {};
+    } catch (e) {
+      _data = {};
+    }
+    if (
+      Object.keys(_data).length === 0 ||
+      !this._decoder ||
+      this.cacheContext.uid !== _data.uid
+    )
+      return;
+    if (this._firstFrame) {
+      for (let renderer of this.renderers) {
+        if (renderer.rendererType !== RendererType.WEBCODECSRENDERER) {
+          continue;
+        }
+        renderer.bind(renderer.context);
+      }
+
+      try {
+        this._decoder.decoderConfigure(_data.videoEncodedFrameInfo);
+      } catch (error: any) {
+        logInfo(error);
+        return;
+      }
+      this._firstFrame = false;
+    }
+    if (this.shouldFallback(_data.videoEncodedFrameInfo)) {
+      AgoraEnv.AgoraRendererManager?.handleWebCodecsFallback(this.cacheContext);
+    } else {
+      this._decoder.decodeFrame(
+        buffer,
+        _data.videoEncodedFrameInfo,
+        new Date().getTime()
+      );
+    }
+  }
+
+  public draw() {
+    if (isUseConnection(this.cacheContext)) {
+      this._engine?.setRemoteVideoSubscriptionOptionsEx(
+        this.cacheContext.uid!,
+        {
+          type: VideoStreamType.VideoStreamHigh,
+          encodedFrameOnly: true,
+        },
+        {
+          channelId: this.cacheContext.channelId,
+          localUid: this.cacheContext.localUid,
+        }
+      );
+    } else {
+      this._engine?.setRemoteVideoSubscriptionOptions(this.cacheContext.uid!, {
+        type: VideoStreamType.VideoStreamHigh,
+        encodedFrameOnly: true,
+      });
+    }
+    AgoraElectronBridge.OnEvent(
+      `call_back_with_encoded_video_frame_${this.cacheContext.uid}`,
+      (...params: any) => {
+        try {
+          this.onEncodedVideoFrameReceived(...params);
+        } catch (e) {
+          console.error(e);
+        }
+      }
+    );
+  }
+
+  public shouldFallback(frameInfo: EncodedVideoFrameInfo): boolean {
+    let shouldFallback = false;
+    if (!frameInfo.codecType) {
+      shouldFallback = true;
+      logInfo('codecType is not supported, fallback to native decoder');
+    } else {
+      const mapping =
+        AgoraEnv.CapabilityManager?.frameCodecMapping[frameInfo.codecType];
+      if (mapping === undefined) {
+        shouldFallback = true;
+        logInfo('codecType is not supported, fallback to native decoder');
+      } else if (
+        mapping.minWidth >= frameInfo.width! &&
+        mapping.minHeight >= frameInfo.height! &&
+        mapping.maxWidth <= frameInfo.width! &&
+        mapping.maxHeight <= frameInfo.height!
+      ) {
+        shouldFallback = true;
+        logInfo('frame size is not supported, fallback to native decoder');
+      }
+    }
+    return shouldFallback;
+  }
+
+  public release(): void {
+    AgoraElectronBridge.UnEvent(
+      `call_back_with_encoded_video_frame_${this.cacheContext.uid}`
+    );
+    this._decoder?.release();
+    this._decoder = null;
+    super.release();
+  }
+
+  public fetchVideoFrame(): { hasMoreFrame: boolean; needRender: boolean } {
+    return { hasMoreFrame: false, needRender: false };
+  }
+
+  public renderFrame(): void {
+    return;
+  }
+
+  public startRendering(): void {
+    return;
+  }
+
+  public stopRendering(): void {
+    return;
+  }
+}