shengwang-electron-sdk 4.6.2-rc.1

package/ts/Renderer/AgoraView.ts

@@ -0,0 +1,264 @@
+import { VideoMirrorModeType, VideoViewSetupMode } from '../Private/AgoraBase';
+import {
+  RenderModeType,
+  VideoModulePosition,
+  VideoSourceType,
+} from '../Private/AgoraMediaBase';
+import { AgoraEnv } from '../Utils';
+
+const VIDEO_SOURCE_TYPE_STRING = 'video-source-type';
+const UID_STRING = 'uid';
+const CHANNEL_ID_STRING = 'channel-id';
+const POSITION_STRING = 'position';
+const RENDERER_CONTENT_MODE_STRING = 'renderer-content-mode';
+const RENDERER_MIRROR_STRING = 'renderer-mirror';
+const ENABLE_ALPHA_MASK_STRING = 'enable-alpha-mask';
+
+const observedAttributes = [
+  VIDEO_SOURCE_TYPE_STRING,
+  UID_STRING,
+  CHANNEL_ID_STRING,
+  POSITION_STRING,
+  RENDERER_CONTENT_MODE_STRING,
+  RENDERER_MIRROR_STRING,
+  ENABLE_ALPHA_MASK_STRING,
+];
+
+declare global {
+  /**
+   * Attributes of the Agora custom element.
+   * You can use this custom element as follows:<agora-view video-source-type="{VideoSourceType.VideoSourceCamera}" channel-id="" uid="{0}" position="{VideoModulePosition.PositionPreRenderer}"></agora-view>
+   */
+  interface AgoraView {
+    /**
+     * The type of the video source. See VideoSourceType .
+     */
+    'video-source-type': VideoSourceType;
+    /**
+     * The ID of the remote user.
+     */
+    'uid': number;
+    /**
+     * The channel name. This parameter signifies the channel in which users engage in real-time audio and video interaction. Under the premise of the same App ID, users who fill in the same channel ID enter the same channel for audio and video interaction. The string length must be less than 64 bytes. Supported characters:
+     * All lowercase English letters: a to z.
+     * All uppercase English letters: A to Z.
+     * All numeric characters: 0 to 9.
+     * Space
+     * "!", "#", "$", "%", "&", "(", ")", "+", "-", ":", ";", "<", "= ", ".", ">", "?", "@", "[", "]", "^", "_", "{", "}", "|", "~", ","
+     */
+    'channel-id': string;
+    /**
+     * The frame position of the video observer.
+     */
+    'position': VideoModulePosition;
+    /**
+     * The video display mode.
+     */
+    'renderer-content-mode': RenderModeType;
+    /**
+     * Whether to enable mirror mode when rendering video: true: Enable mirror mode.false: Do not enable mirror mode.
+     */
+    'renderer-mirror': boolean;
+    /**
+     * The inline style of elements. See style .
+     */
+    'style': any;
+    /**
+     * Whether to enable alpha mask rendering: true : Enable alpha mask rendering. false : (Default) Disable alpha mask rendering. Alpha mask rendering can create images with transparent effects and extract portraits from videos. When used in combination with other methods, you can implement effects such as portrait-in-picture and watermarking.
+     */
+    'enable-alpha-mask': boolean;
+  }
+  namespace JSX {
+    /**
+     * The custom HTML elements defined by Agora.
+     */
+    interface IntrinsicElements {
+      /**
+       * The name of the custom element, which supports rendering an AgoraView or HTMLElement object.
+       */
+      'agora-view': AgoraView | HTMLElement;
+    }
+  }
+}
+
+export default class AgoraView extends HTMLElement {
+  isConnectedCallback = false;
+  static get observedAttributes() {
+    return observedAttributes;
+  }
+
+  get sourceType(): VideoSourceType {
+    const number = Number(this.getAttribute(VIDEO_SOURCE_TYPE_STRING));
+    return isNaN(number) ? 0 : number;
+  }
+
+  set sourceType(val) {
+    if (val) {
+      this.setAttribute(VIDEO_SOURCE_TYPE_STRING, String(val));
+    } else {
+      this.removeAttribute(VIDEO_SOURCE_TYPE_STRING);
+    }
+  }
+
+  get uid(): number {
+    const number = Number(this.getAttribute(UID_STRING));
+    return isNaN(number) ? 0 : number;
+  }
+
+  set uid(val) {
+    if (val) {
+      this.setAttribute(UID_STRING, String(val));
+    } else {
+      this.removeAttribute(UID_STRING);
+    }
+  }
+
+  get channelId(): string {
+    return this.getAttribute(CHANNEL_ID_STRING) || '';
+  }
+
+  set channelId(val) {
+    if (val) {
+      this.setAttribute(CHANNEL_ID_STRING, val);
+    } else {
+      this.removeAttribute(CHANNEL_ID_STRING);
+    }
+  }
+
+  get position(): VideoModulePosition {
+    const number = Number(this.getAttribute(POSITION_STRING));
+    return isNaN(number)
+      ? VideoModulePosition.PositionPreEncoder |
+          VideoModulePosition.PositionPreRenderer
+      : number;
+  }
+
+  set position(val) {
+    if (val) {
+      this.setAttribute(POSITION_STRING, String(val));
+    } else {
+      this.removeAttribute(POSITION_STRING);
+    }
+  }
+
+  get renderMode(): RenderModeType {
+    const number = Number(
+      this.getAttribute(RENDERER_CONTENT_MODE_STRING) ||
+        RenderModeType.RenderModeFit
+    );
+    return isNaN(number) ? RenderModeType.RenderModeFit : number;
+  }
+
+  set renderMode(val) {
+    if (val) {
+      this.setAttribute(RENDERER_CONTENT_MODE_STRING, String(val));
+    } else {
+      this.removeAttribute(RENDERER_CONTENT_MODE_STRING);
+    }
+  }
+
+  get renderMirror(): boolean {
+    return this.getAttribute(RENDERER_MIRROR_STRING) === 'true';
+  }
+
+  set renderMirror(val) {
+    if (val) {
+      this.setAttribute(RENDERER_MIRROR_STRING, String(val));
+    } else {
+      this.removeAttribute(RENDERER_MIRROR_STRING);
+    }
+  }
+
+  get enableAlphaMask(): boolean {
+    return this.getAttribute(ENABLE_ALPHA_MASK_STRING) === 'true';
+  }
+
+  set enableAlphaMask(val) {
+    if (val) {
+      this.setAttribute(ENABLE_ALPHA_MASK_STRING, String(val));
+    } else {
+      this.removeAttribute(ENABLE_ALPHA_MASK_STRING);
+    }
+  }
+
+  constructor() {
+    super();
+  }
+
+  initializeRender = () => {
+    const {
+      channelId,
+      uid,
+      sourceType,
+      position,
+      renderMode,
+      renderMirror,
+      enableAlphaMask,
+    } = this;
+    AgoraEnv.AgoraRendererManager?.addOrRemoveRenderer({
+      sourceType,
+      view: this,
+      uid,
+      channelId,
+      position,
+      renderMode,
+      mirrorMode: renderMirror
+        ? VideoMirrorModeType.VideoMirrorModeEnabled
+        : VideoMirrorModeType.VideoMirrorModeDisabled,
+      setupMode: VideoViewSetupMode.VideoViewSetupReplace,
+      enableAlphaMask,
+    });
+  };
+
+  destroyRender = () => {
+    const { channelId, uid, sourceType, position, enableAlphaMask } = this;
+    AgoraEnv.AgoraRendererManager?.removeRendererFromCache({
+      channelId,
+      position,
+      uid,
+      sourceType,
+      view: this,
+      enableAlphaMask,
+    });
+  };
+
+  connectedCallback() {
+    this.isConnectedCallback = true;
+    this.initializeRender();
+  }
+
+  attributeChangedCallback(attrName: string, oldVal: any, newVal: any) {
+    if (!this.isConnectedCallback) {
+      return;
+    }
+    const isSetRenderOption = [
+      RENDERER_CONTENT_MODE_STRING,
+      RENDERER_MIRROR_STRING,
+      ENABLE_ALPHA_MASK_STRING,
+    ].includes(attrName);
+
+    if (isSetRenderOption) {
+      AgoraEnv.AgoraRendererManager?.setRendererContext({
+        view: this,
+        renderMode: this.renderMode,
+        mirrorMode: this.renderMirror
+          ? VideoMirrorModeType.VideoMirrorModeEnabled
+          : VideoMirrorModeType.VideoMirrorModeDisabled,
+        enableAlphaMask: this.enableAlphaMask,
+      });
+      return;
+    }
+    const isNeedReInitialize = observedAttributes.includes(attrName);
+    if (!isNeedReInitialize) {
+      return;
+    }
+    this.initializeRender();
+  }
+
+  disconnectedCallback() {
+    this.isConnectedCallback = false;
+    this.destroyRender();
+  }
+}
+
+window.customElements.define('agora-view', AgoraView);

package/ts/Renderer/CapabilityManager.ts

@@ -0,0 +1,126 @@
+import semver from 'semver';
+
+import createAgoraRtcEngine from '../AgoraSdk';
+
+import {
+  GpuInfo,
+  VideoDecodeAcceleratorSupportedProfile,
+} from '../Decoder/gpu-utils';
+import { VideoCodecType } from '../Private/AgoraBase';
+import { IRtcEngineEx } from '../Private/IAgoraRtcEngineEx';
+import { ipcSend } from '../Private/ipc/renderer';
+
+import { IPCMessageType, VideoFallbackStrategy, codecMapping } from '../Types';
+import { AgoraEnv, logError, logInfo } from '../Utils';
+
+/**
+ * @ignore
+ */
+export class CapabilityManager {
+  gpuInfo: GpuInfo = new GpuInfo();
+  frameCodecMapping: {
+    [key in VideoCodecType]?: VideoDecodeAcceleratorSupportedProfile;
+  } = {};
+  webCodecsDecoderEnabled: boolean = AgoraEnv.enableWebCodecsDecoder;
+  private _engine: IRtcEngineEx;
+
+  setWebCodecsDecoderEnabled(enabled: boolean): void {
+    this.webCodecsDecoderEnabled = enabled;
+  }
+
+  constructor() {
+    this._engine = createAgoraRtcEngine();
+    if (AgoraEnv.enableWebCodecsDecoder) {
+      this.getGpuInfo(() => {
+        if (
+          AgoraEnv.videoFallbackStrategy ===
+          VideoFallbackStrategy.PerformancePriority
+        ) {
+          if (!this.isSupportedH265()) {
+            if (this.isSupportedH264()) {
+              this._engine.setParameters(
+                JSON.stringify({ 'che.video.h265_dec_enable': false })
+              );
+              logInfo(
+                'the videoFallbackStrategy is PerformancePriority, H265 is not supported, fallback to H264'
+              );
+            } else {
+              this.webCodecsDecoderEnabled = false;
+              logInfo(
+                'the videoFallbackStrategy is PerformancePriority, H264 and H265 are not supported, fallback to native decoder'
+              );
+            }
+          }
+        } else if (
+          AgoraEnv.videoFallbackStrategy ===
+          VideoFallbackStrategy.BandwidthPriority
+        ) {
+          if (!this.isSupportedH265()) {
+            this.webCodecsDecoderEnabled = false;
+            logInfo(
+              'the videoFallbackStrategy is BandwidthPriority, H265 is not supported, fallback to native decoder'
+            );
+          }
+        }
+      });
+    }
+  }
+
+  public getGpuInfo(callback?: () => void): void {
+    //getGpuInfo and videoDecoder is not supported in electron version < 22.0.0
+    //@ts-ignore
+    if (semver.lt(process.versions.electron, '22.0.0')) {
+      logError(
+        'WebCodecsDecoder is not supported in electron version < 22.0.0, please upgrade electron to 22.0.0 or later.'
+      );
+      return;
+    }
+    //@ts-ignore
+    if (process.type === 'renderer') {
+      ipcSend(IPCMessageType.AGORA_IPC_GET_GPU_INFO)
+        .then((result) => {
+          this.gpuInfo.videoDecodeAcceleratorSupportedProfile = result;
+          this.webCodecsDecoderEnabled = (AgoraEnv.enableWebCodecsDecoder &&
+            this.gpuInfo.videoDecodeAcceleratorSupportedProfile.length > 0)!;
+
+          result.forEach((profile: VideoDecodeAcceleratorSupportedProfile) => {
+            const match = codecMapping.find((item) =>
+              profile.codec.includes(item.profile)
+            );
+            if (match) {
+              //Normally, the range of compatible widths and heights should be the same under the same codec.
+              //there is no need to differentiate between different profiles. This could be optimized in the future.
+              this.frameCodecMapping[match.type] = {
+                codec: match.codec,
+                minWidth: profile.minWidth,
+                minHeight: profile.minHeight,
+                maxWidth: profile.maxWidth,
+                maxHeight: profile.maxHeight,
+              };
+            }
+          });
+          callback && callback();
+        })
+        .catch((error) => {
+          logError(
+            'Failed to get GPU info, please check if you are already import agora-electron-sdk in the main process.',
+            error
+          );
+        });
+    } else {
+      logError('This function only works in renderer process');
+    }
+  }
+
+  public isSupportedH264(): boolean {
+    return this.frameCodecMapping[VideoCodecType.VideoCodecH264] !== undefined;
+  }
+
+  public isSupportedH265(): boolean {
+    return this.frameCodecMapping[VideoCodecType.VideoCodecH265] !== undefined;
+  }
+
+  release(): void {
+    AgoraEnv.enableWebCodecsDecoder = false;
+  }
+}

package/ts/Renderer/IRenderer.ts

@@ -0,0 +1,163 @@
+import { VideoMirrorModeType } from '../Private/AgoraBase';
+import { RenderModeType, VideoFrame } from '../Private/AgoraMediaBase';
+import { CodecConfigInfo, RendererContext, RendererType } from '../Types';
+
+export abstract class IRenderer {
+  parentElement?: HTMLElement;
+  container?: HTMLElement;
+  canvas?: HTMLCanvasElement;
+  rendererType: RendererType | undefined;
+  context: RendererContext = {};
+  private _frameCount = 0;
+  private _startTime: number | null = null;
+
+  public bind(context: RendererContext) {
+    this.parentElement = context.view;
+    this.container = document.createElement('div');
+    Object.assign(this.container.style, {
+      width: '100%',
+      height: '100%',
+      display: 'flex',
+      justifyContent: 'center',
+      alignItems: 'center',
+      overflow: 'hidden',
+    });
+    this.parentElement?.appendChild(this.container);
+    this.canvas = document.createElement('canvas');
+    this.canvas.style.display = 'none';
+    this.container.appendChild(this.canvas);
+  }
+
+  public unbind() {
+    if (this.container && this.canvas?.parentNode === this.container) {
+      this.container.removeChild(this.canvas);
+    }
+    if (
+      this.parentElement &&
+      this.container?.parentNode === this.parentElement
+    ) {
+      this.parentElement.removeChild(this.container);
+    }
+
+    this.canvas = undefined;
+    this.container = undefined;
+    this.parentElement = undefined;
+  }
+
+  public drawFrame(
+    uid: number,
+    _videoFrame?: VideoFrame,
+    _codecConfig?: CodecConfigInfo
+  ): void {
+    if (!this.canvas) return;
+    if (this.canvas.style.display !== '') {
+      this.canvas.style.display = '';
+    }
+  }
+
+  public setContext(context: RendererContext) {
+    if (this.context.renderMode !== context.renderMode) {
+      this.context.renderMode = context.renderMode;
+      this.updateRenderMode();
+    }
+
+    if (this.context.mirrorMode !== context.mirrorMode) {
+      this.context.mirrorMode = context.mirrorMode;
+      this.updateMirrorMode();
+    }
+
+    if (this.context.enableAlphaMask !== context.enableAlphaMask) {
+      this.context.enableAlphaMask = context.enableAlphaMask;
+    }
+    this.context = { ...this.context, ...context };
+  }
+
+  protected updateRenderMode(): void {
+    if (!this.canvas || !this.container) return;
+
+    const { clientWidth, clientHeight } = this.container;
+    const { width, height } = this.canvas;
+
+    const containerAspectRatio = clientWidth / clientHeight;
+    const canvasAspectRatio = width / height;
+    const widthScale = clientWidth / width;
+    const heightScale = clientHeight / height;
+    const isHidden =
+      this.context.renderMode === RenderModeType.RenderModeHidden;
+
+    let scale = 1;
+    // If container's aspect ratio is larger than canvas's aspect ratio
+    if (containerAspectRatio > canvasAspectRatio) {
+      // Scale canvas to fit container's width on hidden mode
+      // Scale canvas to fit container's height on fit mode
+      scale = isHidden ? widthScale : heightScale;
+    } else {
+      // Scale canvas to fit container's height on hidden mode
+      // Scale canvas to fit container's width on fit mode
+      scale = isHidden ? heightScale : widthScale;
+    }
+    this.canvas.style.transform = `scale(${scale})`;
+  }
+
+  protected updateMirrorMode(): void {
+    if (!this.parentElement) return;
+
+    Object.assign(this.parentElement.style, {
+      transform:
+        this.context.mirrorMode === VideoMirrorModeType.VideoMirrorModeEnabled
+          ? 'rotateY(180deg)'
+          : '',
+    });
+  }
+
+  protected rotateCanvas({ width, height, rotation }: VideoFrame): void {
+    if (!this.canvas) return;
+
+    if (rotation === 0 || rotation === 180) {
+      this.canvas.width = width!;
+      this.canvas.height = height!;
+    } else if (rotation === 90 || rotation === 270) {
+      this.canvas.height = width!;
+      this.canvas.width = height!;
+    } else {
+      throw new Error(
+        `Invalid rotation: ${rotation}, only 0, 90, 180, 270 are supported`
+      );
+    }
+  }
+
+  public getFps(): number {
+    let fps = 0;
+    if (!this.context.enableFps || !this.container) {
+      return fps;
+    }
+    if (this._startTime == null) {
+      this._startTime = performance.now();
+    } else {
+      const elapsed = (performance.now() - this._startTime) / 1000;
+      fps = ++this._frameCount / elapsed;
+    }
+
+    let span = this.container.querySelector('span');
+    if (!span) {
+      span = document.createElement('span');
+
+      Object.assign(span.style, {
+        position: 'absolute',
+        bottom: '0',
+        left: '0',
+        zIndex: '10',
+        width: '55px',
+        background: '#fff',
+      });
+
+      this.container.style.position = 'relative';
+
+      this.container.appendChild(span);
+    }
+
+    span.innerText = `fps: ${fps.toFixed(0)}`;
+
+    return fps;
+  }
+}

package/ts/Renderer/IRendererCache.ts

@@ -0,0 +1,130 @@
+import { VideoSourceType } from '../Private/AgoraMediaBase';
+import { RtcConnection } from '../Private/IAgoraRtcEngineEx';
+
+import { RendererCacheContext, RendererContext } from '../Types';
+
+import { IRenderer } from './IRenderer';
+
+/**
+ * @ignore
+ */
+export function generateRendererCacheKey({
+  channelId,
+  uid,
+  sourceType,
+}: RendererContext): string {
+  return `${channelId}_${uid}_${sourceType}`;
+}
+
+/**
+ * @ignore
+ */
+export function isUseConnection(context: RendererCacheContext): boolean {
+  // if RtcConnection is not undefined, then use connection
+  return !!context.channelId && context.localUid !== undefined;
+}
+
+export abstract class IRendererCache {
+  renderers: IRenderer[];
+  cacheContext: RendererCacheContext;
+  callbackContext: { connection: RtcConnection; sourceType: VideoSourceType };
+
+  // 性能统计数据
+  public actualFps: number = 0;
+  public avgFrameTime: number = 0;
+  public maxFrameTime: number = 0;
+  public minFrameTime: number = 0;
+  public avgFrameInterval: number = 0;
+  public maxFrameInterval: number = 0;
+  public minFrameInterval: number = 0;
+
+  constructor({
+    channelId,
+    uid,
+    useWebCodecsDecoder,
+    enableFps,
+    sourceType,
+    localUid,
+    position,
+    enableAlphaMask,
+  }: RendererContext) {
+    this.renderers = [];
+    this.cacheContext = {
+      channelId,
+      uid,
+      useWebCodecsDecoder,
+      enableFps,
+      sourceType,
+      localUid,
+      position,
+      enableAlphaMask,
+    };
+    this.callbackContext = {
+      connection: { channelId, localUid },
+      sourceType: sourceType!,
+    };
+  }
+
+  public setCallbackContext(
+    connection: RtcConnection,
+    sourceType: VideoSourceType
+  ): void {
+    this.callbackContext = { connection, sourceType };
+  }
+
+  public get key(): string {
+    return generateRendererCacheKey(this.cacheContext);
+  }
+
+  public abstract draw(): void;
+
+  public findRenderer(view: Element): IRenderer | undefined {
+    return this.renderers.find((renderer) => renderer.parentElement === view);
+  }
+
+  public addRenderer(renderer: IRenderer): void {
+    this.renderers.push(renderer);
+  }
+
+  /**
+   * Remove the specified renderer if it is specified, otherwise remove all renderers
+   */
+  public removeRenderer(renderer?: IRenderer): void {
+    let start = 0;
+    let deleteCount = this.renderers.length;
+    if (renderer) {
+      start = this.renderers.indexOf(renderer);
+      if (start < 0) return;
+      deleteCount = 1;
+    }
+    this.renderers.splice(start, deleteCount).forEach((it) => it.unbind());
+  }
+
+  public setRendererContext(context: RendererContext): boolean {
+    if (context.view) {
+      const renderer = this.findRenderer(context.view);
+      if (renderer) {
+        renderer.setContext(context);
+        return true;
+      }
+      return false;
+    } else {
+      this.renderers.forEach((it) => {
+        it.setContext(context);
+      });
+      return this.renderers.length > 0;
+    }
+  }
+
+  public abstract fetchVideoFrame(): {
+    hasMoreFrame: boolean;
+    needRender: boolean;
+  };
+  public abstract renderFrame(): void;
+  public abstract startRendering(): void;
+  public abstract stopRendering(): void;
+
+  public release(): void {
+    this.removeRenderer();
+  }
+}