shengwang-electron-sdk 4.6.2-rc.1

package/ts/Decoder/index.ts

@@ -0,0 +1,210 @@
+import { EncodedVideoFrameInfo, VideoFrameType } from '../Private/AgoraBase';
+
+import { WebCodecsRenderer } from '../Renderer/WebCodecsRenderer/index';
+import { CodecConfigInfo, RendererCacheContext, RendererType } from '../Types';
+import { AgoraEnv, logDebug, logInfo } from '../Utils';
+
+const frameTypeMapping = {
+  [VideoFrameType.VideoFrameTypeDeltaFrame]: 'delta',
+  [VideoFrameType.VideoFrameTypeKeyFrame]: 'key',
+  [VideoFrameType.VideoFrameTypeDroppableFrame]: 'delta', // this is a workaround for the issue that the frameType is not correct
+};
+
+export class WebCodecsDecoder {
+  private _decoder: VideoDecoder;
+  private renderers: WebCodecsRenderer[] = [];
+  private _cacheContext: RendererCacheContext;
+  // eslint-disable-next-line auto-import/auto-import
+  private pendingFrame: VideoFrame | null = null;
+  private _currentCodecConfig: CodecConfigInfo | null = null;
+
+  private _base_ts = 0;
+  private _base_ts_ntp = 1;
+  private _last_ts_ntp = 1;
+  private _decode_retry_count = 0;
+
+  constructor(
+    renders: WebCodecsRenderer[],
+    onError: (e: any) => void,
+    context: RendererCacheContext
+  ) {
+    this.renderers = renders;
+    this._cacheContext = context;
+    this._decoder = new VideoDecoder({
+      // @ts-ignore
+      output: this._output.bind(this),
+      error: (e) => {
+        onError(e);
+      },
+    });
+  }
+
+  // eslint-disable-next-line auto-import/auto-import
+  _output(frame: VideoFrame) {
+    // Schedule the frame to be rendered.
+    this._renderFrame(frame);
+  }
+
+  // eslint-disable-next-line auto-import/auto-import
+  private _renderFrame(frame: VideoFrame) {
+    if (!this.pendingFrame) {
+      // Schedule rendering in the next animation frame.
+      // eslint-disable-next-line auto-import/auto-import
+      requestAnimationFrame(this.renderAnimationFrame.bind(this));
+    } else {
+      // Close the current pending frame before replacing it.
+      this.pendingFrame.close();
+    }
+    // Set or replace the pending frame.
+    this.pendingFrame = frame;
+  }
+
+  renderAnimationFrame() {
+    for (let renderer of this.renderers) {
+      if (renderer.rendererType !== RendererType.WEBCODECSRENDERER) {
+        continue;
+      }
+      renderer.drawFrame(
+        this._cacheContext.uid!,
+        this.pendingFrame,
+        this._currentCodecConfig!
+      );
+      this.pendingFrame = null;
+    }
+  }
+
+  decoderConfigure(frameInfo: EncodedVideoFrameInfo) {
+    this.pendingFrame = null;
+    // @ts-ignore
+    let codec =
+      AgoraEnv.CapabilityManager?.frameCodecMapping[frameInfo.codecType!]
+        ?.codec;
+    if (!codec) {
+      AgoraEnv.AgoraRendererManager?.handleWebCodecsFallback(
+        this._cacheContext
+      );
+      throw new Error(
+        `codec ${frameInfo.codecType} is not in frameCodecMapping,failed to configure decoder, fallback to native decoder`
+      );
+    }
+    this._currentCodecConfig = {
+      codecType: frameInfo.codecType,
+      codedWidth: frameInfo.width,
+      codedHeight: frameInfo.height,
+      rotation: frameInfo.rotation,
+    };
+    this._decoder!.configure({
+      codec: codec,
+      codedWidth: frameInfo.width,
+      codedHeight: frameInfo.height,
+    });
+    logInfo(
+      `configure decoder: codedWidth: ${frameInfo.width}, codedHeight: ${frameInfo.height},codec: ${codec}`
+    );
+  }
+
+  updateTimestamps(ts: number) {
+    if (this._base_ts !== 0) {
+      if (ts > this._base_ts) {
+        this._last_ts_ntp =
+          this._base_ts_ntp + Math.floor(((ts - this._base_ts) * 1000) / 90);
+      } else {
+        this._base_ts = ts;
+        this._last_ts_ntp++;
+        this._base_ts_ntp = this._last_ts_ntp;
+      }
+    } else {
+      this._base_ts = ts;
+      this._last_ts_ntp = 1;
+    }
+  }
+
+  handleCodecIsChanged(frameInfo: EncodedVideoFrameInfo) {
+    if (
+      this._currentCodecConfig?.codecType !== frameInfo.codecType ||
+      this._currentCodecConfig?.codedWidth !== frameInfo.width ||
+      this._currentCodecConfig?.codedHeight !== frameInfo.height ||
+      this._currentCodecConfig?.rotation !== frameInfo.rotation
+    ) {
+      logInfo('frameInfo has changed, reconfigure decoder');
+      this._decoder.reset();
+      this.decoderConfigure(frameInfo);
+    }
+  }
+
+  // @ts-ignore
+  decodeFrame(
+    imageBuffer: Uint8Array,
+    frameInfo: EncodedVideoFrameInfo,
+    ts: number
+  ) {
+    try {
+      this.handleCodecIsChanged(frameInfo);
+    } catch (error: any) {
+      logInfo(error);
+      return;
+    }
+
+    if (!imageBuffer) {
+      logDebug('imageBuffer is empty, skip decode frame');
+      return;
+    }
+
+    let frameType: string | undefined;
+    if (frameInfo.frameType !== undefined) {
+      // @ts-ignore
+      frameType = frameTypeMapping[frameInfo.frameType];
+    }
+    if (!frameType) {
+      logDebug('frameType is not in frameTypeMapping, skip decode frame');
+      return;
+    }
+
+    this.updateTimestamps(ts);
+
+    try {
+      this._decoder.decode(
+        new EncodedVideoChunk({
+          data: imageBuffer,
+          timestamp: this._last_ts_ntp,
+          // @ts-ignore
+          type: frameType,
+          // @ts-ignore
+          transfer: [imageBuffer.buffer],
+        })
+      );
+      this._decode_retry_count = 0;
+    } catch (e) {
+      /// There are some cases that the decoder failed to decode the frame, we will retry for a few times
+      /// If the decoder still failed to decode the frame, we will fallback to native decoder
+      /// The retry count is defined in AgoraEnv.maxDecodeRetryCount
+      /// The retry count will be reset to 0 when the decoder successfully decode a frame
+      if (this._decode_retry_count >= AgoraEnv.maxDecodeRetryCount) {
+        AgoraEnv.AgoraRendererManager?.handleWebCodecsFallback(
+          this._cacheContext
+        );
+        throw new Error(
+          `failed to decode frame over ${this._decode_retry_count} times, fallback to native decoder`
+        );
+      }
+      this._decode_retry_count++;
+    }
+  }
+
+  reset() {
+    this._base_ts = 0;
+    this._base_ts_ntp = 1;
+    this._last_ts_ntp = 1;
+    this._decoder.reset();
+  }
+
+  release() {
+    try {
+      if (this.pendingFrame) {
+        this.pendingFrame.close();
+      }
+      this._decoder.close();
+    } catch (e) {}
+    this.pendingFrame = null;
+  }
+}