yjz-web-sdk 1.0.0

package/README.md

@@ -0,0 +1,489 @@
+# yjz-web-sdk
+
+<div align=center>
+<img src=public/vite.svg/>
+</div>
+
+针对于亚矩阵项目的云手机投屏和屏幕控制的插件
+
+## 功能支持
+* 支持亚矩阵平台webrtc协议的投屏
+* 支持连接成功后远程视频流和音频流的回调
+* 支持屏幕触控事件，包含电脑模式和手机模式（点击、移动、长按等）
+* 支持电脑键盘映射功能（C+V、C+X、C+Z等）
+* 支持手机端的常用快捷键（back、home、recent、音量等）
+* 支持连接错误回调（分不通错误码返回）
+* 支持webrtc的推拉流实时信息统计
+* 支持webrtc连接状态的回调
+* 支持Vue3
+
+> 一句话描述SDK：
+>
+> 没有纯用户界面，都是使用的webrtc协议，纯js实现，没有依赖任何第三方库，可以很方便的集成到项目中，并且支持Vue3
+
+
+
+```shell
+#  install for vue3
+npm install yjz-web-sdk --save 
+```
+
+## 远程音视频流回调
+
+```ts
+sdk.value.on(EmitType.streamTrack, (track: MediaStreamTrack) => {
+    if (!remoteVideoElement.value) return;
+    if (!remoteVideoElement.value.srcObject) {
+        remoteVideoElement.value.srcObject = new MediaStream();
+    }
+    remoteVideoElement.value.playsInline = true;
+    remoteVideoElement.value.setAttribute('webkit-playsinline', 'true');
+    (remoteVideoElement.value.srcObject as MediaStream).addTrack(track);
+});
+```
+
+## Webrtc连接状态
+
+```ts
+type RTCIceConnectionState = "checking" | "closed" | "completed" | "connected" | "disconnected" | "failed" | "new";
+
+sdk.value.on(EmitType.iceConnectionState, (state: string) => {
+    console.log("iceConnectionState====>", state);
+});
+```
+
+## Webrtc统计信息
+| 左对齐 | 右对齐        |
+| :-----|:-----------|
+| connectionType | webrtc连接模式 |
+| framesPerSecond | 每秒帧率       |
+| currentRoundTripTime | 每秒帧率   |
+| lostRate | 丢包率        |
+| bitrate | 视频码率       |
+| averageDecodeTime | 解码时间       |
+
+```ts
+sdk.value.on(EmitType.statisticInfo, (info: any) => {
+console.log("statisticInfo====>", info);
+});
+```
+
+## 错误回调
+| 错误码   | 左对齐           | 等级 |
+|:------|:--------------|:--:|
+| 10001 | WebSocket连接失败 | 高  |
+| 10002 | WebSocket已关闭  | 高  |
+| 10003 | 创建offer失败     | 高  |
+| 10004 | 处理offer失败     | 高  |
+| 10005 | 创建answer失败    | 高  |
+| 10006 | 处理answer失败    | 高  |
+| 10007 | 设置本地描述失败      | 高  |
+| 10008 | 设置远端描述失败      | 高  |
+| 10009 | 处理 ICE 交换失败   | 高  |
+| 10010 | ICE 状态异常           | 高  |
+| 10011 | 摄像头异常           | 高  |
+| 10012 | 云机已关机或者不存在           | 高  |
+| 10013 | 信令通道错误           | 低  |
+| 10014 | webrtc统计信息获取失败           | 低  |
+
+```ts
+sdk.value.on(EmitType.webrtcError, (error: any) => {
+    console.log("webrtcError====>", error);
+});
+```
+
+
+**Vue3**
+使用示例
+
+```vue
+<template>
+  <div ref="videoFull" class="flex" style="height: 100vh;background: black;position: relative;width:100vw">
+    <div ref="videoContainer" class="flex flex-1 items-center justify-center" style="position:relative;overflow:hidden">
+      <div ref="keyboardArea" @mouseenter="startListening" @mouseleave="stopListening" tabindex="0"
+           class="vContainer"
+           :style="{height:videoSize.height + 'px', width:videoSize.width + 'px', transform: `rotate(${videoAngle}deg)`}" style="position: relative">
+        <video ref="remoteVideoElement" @touchstart="onTouchStart"
+               @touchmove="onTouchMove"
+               @touchend="onTouchEnd" @pointerenter="handleMouseEnter"
+               @pointerdown="handleMouseDown" @pointermove="handleMouseMove"
+               @pointerup="handleMouseUp" @pointerleave="handleMouseLeave" id="phoneVideo" style="width: 100%;height: 100%;" autoplay playsinline muted :disablePictureInPicture="true"></video>
+      </div>
+    </div>
+  </div>
+</template>
+
+<script setup lang="ts">
+import { ref, onMounted, onUnmounted, computed } from "vue";
+import {
+  WebRTCSdk,
+  getKeyEventData,
+  transformCoordinate,
+  valueToPercentage,
+  ActionType,
+  ChannelDataType,
+  InputData,
+  KeyEventData,
+  TouchData,
+  ContainerDirection,
+  EmitType
+} from 'yjz-web-sdk';
+
+interface Dimension {
+  width: number;
+  height: number;
+}
+
+const dimensions = ref<Dimension>({ width: 0, height: 0 });
+const videoContainer = ref<HTMLElement | null>(null);
+const videoFull = ref<HTMLElement | null>(null);
+const screenStatus = ref(false);
+const isMobile = ref(false);
+const pointerList = ref<number[]>(new Array(20).fill(0));
+const pointerDownTime = ref<number[]>(new Array(10).fill(0));
+const bound = ref(100);
+const remoteVideoElement = ref<HTMLVideoElement | null>(null);
+const isDragging = ref(false);
+const widthRadio = ref(0);
+const heightRadio = ref(0);
+const sdk = ref<WebRTCSdk | null>(null);
+const streamAngle = ref(0);
+const videoAngle = ref(0);
+const cloudDeviceSize = ref<Dimension>({ width: 0, height: 0 });
+const resizeObserver = ref<ResizeObserver | null>(null);
+const remoteVideo = ref<Partial<Dimension>>({});
+const videoDimension = ref<Dimension>({ width: 0, height: 0 });
+const connectInfo = ref('');
+const listening = ref(false);
+const keyboardArea = ref<HTMLElement | null>(null);
+const isComposing = ref(false);
+const endX = ref(0);
+const endY = ref(0);
+
+const sendKeyBoardData = (value: { androidKeyCode: number; metaState: number }) => {
+  if (sdk.value) {
+    const keyEvent = new KeyEventData(value.androidKeyCode, value.metaState);
+    sdk.value.sendChannelData(ChannelDataType.ActionInput, keyEvent);
+  }
+};
+
+const sendKeyBoardData1 = (key: number) => {
+  if (sdk.value) {
+    const keyEvent = new KeyEventData(key, 0);
+    sdk.value.sendChannelData(ChannelDataType.ActionInput, keyEvent);
+  }
+};
+
+const sendChineseData = (value: string) => {
+  if (sdk.value) {
+    const inputData = new InputData(value);
+    sdk.value.sendChannelData(ChannelDataType.ActionChinese, inputData);
+  }
+};
+
+const startListening = () => {
+  listening.value = true;
+  document.addEventListener('keydown', handleKeyDown);
+};
+
+const stopListening = () => {
+  listening.value = false;
+  document.removeEventListener('keydown', handleKeyDown);
+};
+
+const handleKeyDown = (event: KeyboardEvent) => {
+  if (!listening.value || isComposing.value) return;
+  const androidEvent = getKeyEventData(event);
+  if (androidEvent.androidKeyCode === -1) return;
+  sendKeyBoardData(androidEvent);
+};
+
+const onTouchStart = (event: TouchEvent) => {
+  if (isMobile.value) {
+    const touches = Array.from(event.touches);
+    if (touches.length > 0) {
+      pointerDownTime.value[0] = event.timeStamp;
+      bound.value = Math.trunc(4 / widthRadio.value);
+      endX.value = touches[0].clientX;
+      endY.value = touches[0].clientY;
+      createTouchPacket({
+        clientX: endX.value,
+        clientY: endY.value,
+        timeStamp: event.timeStamp,
+      }, ActionType.ACTION_DOWN, 0);
+    }
+  }
+};
+
+const onTouchMove = (event: TouchEvent) => {
+  if (isMobile.value) {
+    const touches = Array.from(event.touches);
+    if (touches.length > 0) {
+      endX.value = touches[0].clientX;
+      endY.value = touches[0].clientY;
+      createTouchPacket({
+        clientX: endX.value,
+        clientY: endY.value,
+        timeStamp: event.timeStamp,
+      }, ActionType.ACTION_MOVE, 0);
+    }
+  }
+};
+
+const onTouchEnd = (event: TouchEvent) => {
+  if (isMobile.value) {
+    createTouchPacket({
+      clientX: endX.value,
+      clientY: endY.value,
+      timeStamp: event.timeStamp,
+    }, ActionType.ACTION_UP, 0);
+  }
+};
+
+const handleMouseDown = (event: MouseEvent) => {
+  if (isMobile.value) return;
+  isDragging.value = true;
+  pointerDownTime.value[0] = event.timeStamp;
+  bound.value = Math.trunc(4 / widthRadio.value);
+  createTouchPacket(event, ActionType.ACTION_DOWN, 0);
+};
+
+const handleMouseMove = (event: MouseEvent) => {
+  if (!isDragging.value || isMobile.value) return;
+  createTouchPacket(event, ActionType.ACTION_MOVE, 0);
+};
+
+const handleMouseEnter = (event: MouseEvent) => {
+  if (isMobile.value) return;
+  isDragging.value = true;
+};
+
+const handleMouseUp = (event: MouseEvent) => {
+  if (!isDragging.value || isMobile.value) return;
+  isDragging.value = false;
+  createTouchPacket(event, ActionType.ACTION_UP, 0);
+};
+
+const handleMouseLeave = (event: MouseEvent) => {
+  if (!isDragging.value) return;
+  isDragging.value = false;
+};
+
+const createTouchPacket = (
+    event: MouseEvent | { clientX: number; clientY: number; timeStamp: number },
+    action: number,
+    p: number
+) => {
+  if (!remoteVideoElement.value) return;
+
+  const offsetTime = Math.trunc((event.timeStamp - pointerDownTime.value[0]) / 2);
+  const rect = remoteVideoElement.value.getBoundingClientRect();
+  let x = event.clientX - rect.left;
+  let y = event.clientY - rect.top;
+
+  const cloudWidth = cloudDeviceSize.value.width;
+  const cloudHeight = cloudDeviceSize.value.height;
+  const result = transformCoordinate(
+      rect.width, rect.height, cloudWidth, cloudHeight,
+      videoAngle.value, streamAngle.value, x, y
+  );
+
+  if (!result || result.length < 2) return;
+  x = result[0];
+  y = result[1];
+
+  if (action === ActionType.ACTION_MOVE) {
+    const flipY = pointerList.value[10 + p] - y;
+    const flipX = pointerList.value[p] - x;
+    if (Math.abs(flipY) < bound.value && Math.abs(flipX) < bound.value) return;
+  }
+
+  pointerList.value[p] = x;
+  pointerList.value[10 + p] = y;
+
+  const percentResult = valueToPercentage(
+      rect.width, rect.height, cloudWidth, cloudHeight,
+      videoAngle.value, streamAngle.value, x, y
+  );
+
+  createTouchEvent(action, p, percentResult[0], percentResult[1], offsetTime);
+};
+
+const createTouchEvent = (action: number, p: number, x: number, y: number, offsetTime: number) => {
+  const touchData = new TouchData(action, p, x, y, offsetTime);
+  sendChannelData(touchData);
+};
+
+const videoSize = computed<Dimension>(() => {
+  let width = 0;
+  let height = 0;
+  if (dimensions.value.width > 0 && dimensions.value.height > 0) {
+    const isRotated = videoAngle.value % 180 !== 0;
+    const containerWidth = isRotated ? dimensions.value.height : dimensions.value.width;
+    const containerHeight = isRotated ? dimensions.value.width : dimensions.value.height;
+
+    if (remoteVideo.value.width && remoteVideo.value.height) {
+      const aspect = remoteVideo.value.width / remoteVideo.value.height;
+      const videoHeight = containerWidth / aspect;
+      if (videoHeight > containerHeight) {
+        height = containerHeight;
+        width = height * aspect;
+      } else {
+        width = containerWidth;
+        height = videoHeight;
+      }
+      widthRadio.value = remoteVideo.value.width / width;
+      heightRadio.value = remoteVideo.value.height / height;
+    }
+  }
+
+  videoDimension.value = { width, height };
+  return { width, height };
+});
+
+const initVideoContainer = () => {
+  resizeObserver.value = new ResizeObserver((entries) => {
+    for (const entry of entries) {
+      const { width, height } = entry.contentRect;
+      dimensions.value = { width, height };
+    }
+  });
+
+  if (videoContainer.value) {
+    resizeObserver.value.observe(videoContainer.value);
+  }
+
+  if (remoteVideoElement.value) {
+    remoteVideoElement.value.addEventListener('resize', () => {
+      if (!remoteVideoElement.value) return;
+      const actualWidth = remoteVideoElement.value.videoWidth;
+      const actualHeight = remoteVideoElement.value.videoHeight;
+      remoteVideo.value = { width: actualWidth, height: actualHeight };
+      screenStatus.value = true;
+    });
+  }
+};
+
+const startConnect = () => {
+  const config = {
+    signalServerUrl: "ws://cs11.nuliapp.com/v1/signaling",
+    roomId: "io_service_13",
+    turnServerUri: "turn:btweb.yajuzhen.com:3478"
+  };
+  sdk.value = new WebRTCSdk(config);
+  sdk.value.startConnection();
+
+  sdk.value.on(EmitType.streamTrack, (track: MediaStreamTrack) => {
+    if (!remoteVideoElement.value) return;
+    if (!remoteVideoElement.value.srcObject) {
+      remoteVideoElement.value.srcObject = new MediaStream();
+    }
+    remoteVideoElement.value.playsInline = true;
+    remoteVideoElement.value.setAttribute('webkit-playsinline', 'true');
+    (remoteVideoElement.value.srcObject as MediaStream).addTrack(track);
+  });
+
+  sdk.value.on(EmitType.iceConnectionState, (state: string) => {
+    console.log("iceConnectionState====>", state);
+  });
+
+  sdk.value.on(EmitType.cloudStatusChanged, (cloudStatus: any) => {
+    console.error("cloudStatusChanged====>", cloudStatus);
+    streamAngle.value = cloudStatus.direction === ContainerDirection.Vertical ? 0 : -90;
+    cloudDeviceSize.value = {
+      width: cloudStatus.screenWidth,
+      height: cloudStatus.screenHeight
+    };
+  });
+
+  sdk.value.on(EmitType.webrtcError, (error: any) => {
+    console.log("webrtcError====>", error);
+  });
+
+  sdk.value.on(EmitType.statisticInfo, (info: any) => {
+    console.log("statisticInfo====>", info);
+  });
+};
+
+const stopConnect = () => {
+  screenStatus.value = false;
+  if (sdk.value) {
+    sdk.value.stop();
+    sdk.value = null;
+  }
+
+  if (remoteVideoElement.value) {
+    remoteVideoElement.value.removeEventListener("resize", () => {});
+    if (remoteVideoElement.value.srcObject) {
+      (remoteVideoElement.value.srcObject as MediaStream).getTracks().forEach(track => track.stop());
+    }
+    remoteVideoElement.value.srcObject = null;
+  }
+
+  connectInfo.value = "";
+};
+
+const sendChannelData = (data: TouchData) => {
+  if (sdk.value) {
+    sdk.value.sendChannelData(ChannelDataType.ClickData, data);
+  }
+};
+
+const checkDeviceMode = () => {
+  isMobile.value = "ontouchstart" in window || navigator.maxTouchPoints > 0;
+};
+
+const checkTouchSupport = (event: PointerEvent) => {
+  isMobile.value = event.pointerType === "touch";
+};
+
+const handleClick = () => {
+  if (remoteVideoElement.value?.srcObject) {
+    remoteVideoElement.value.muted = false;
+  }
+};
+
+onMounted(() => {
+  checkDeviceMode();
+  window.addEventListener("resize", checkDeviceMode);
+  window.addEventListener("pointerdown", checkTouchSupport);
+  initVideoContainer();
+  startConnect();
+  document.addEventListener("click", handleClick);
+});
+
+onUnmounted(() => {
+  stopConnect();
+  document.removeEventListener("click", handleClick);
+  window.removeEventListener("resize", checkDeviceMode);
+  window.removeEventListener("pointerdown", checkTouchSupport);
+  if (resizeObserver.value) {
+    resizeObserver.value.disconnect();
+    resizeObserver.value = null;
+  }
+});
+
+
+
+</script>
+
+<style scoped>
+.vContainer{
+  transition: transform  .2s linear
+}
+
+.flex {
+  display: flex;
+}
+.flex-1 {
+  flex: 1 1 0%;
+}
+.items-center {
+  align-items: center;
+}
+.justify-center {
+  justify-content: center;
+}
+
+</style>
+```

package/lib/core/WebRTCSdk.d.ts

@@ -0,0 +1,33 @@
+import { EventEmitter } from "eventemitter3";
+import { SignalingClient } from "./signal/SignalingClient.ts";
+import { WebRTCClient } from "./rtc/WebRTCClient.ts";
+import { WebRTCConfig } from "./rtc/WebRTCConfig.ts";
+import type { WebRTCConfigOptions } from './rtc/WebRTCConfig';
+import { ChannelDataType } from "./data/WebrtcDataType.ts";
+export declare class WebRTCSdk extends EventEmitter {
+    config: WebRTCConfig;
+    signalingClient: SignalingClient;
+    webRTCClient: WebRTCClient;
+    constructor(options: WebRTCConfigOptions);
+    /** 开始连接 signal 服务 */
+    startConnection(): void;
+    /** 停止连接，并发送退出信令 */
+    stop(): void;
+    /** 开始推流：触发媒体采集与轨道添加 */
+    startPush(): Promise<void>;
+    /** 发送信道数据 */
+    sendChannelData(type: ChannelDataType, data: any): void;
+    /**
+     * 处理 signal 消息，根据不同消息类型分发到 webRTCClient 或直接触发 SDK 事件
+     * @param message 信令消息
+     */
+    private handleSignaling;
+    /** 发送 Offer 信令 */
+    sendOffer: (offerSdp: string) => void;
+    /** 发送 Answer 信令 */
+    sendAnswer: (answerSdp: string) => void;
+    /** 发送 ICE 候选信息 */
+    sendICEMessage: (ice: string) => void;
+    sendPong(): void;
+    sendSignOut(): void;
+}

package/lib/core/data/MessageType.d.ts

@@ -0,0 +1,28 @@
+export declare enum MessageType {
+    Peers = "Peers",
+    Offer = "Offer",
+    Answer = "Answer",
+    IceCandidates = "IceCandidates",
+    SignOut = "SignOut",
+    NotAvailable = "NotAvailable",
+    Ping = "Ping",
+    Pong = "Pong"
+}
+export declare enum SendType {
+    SignIn = "SignIn",
+    SignOut = "SignOut",
+    Answer = "Answer",
+    Offer = "Offer",
+    IceCandidates = "IceCandidates",
+    Pong = "Pong",
+    Identity = "Web"
+}
+export declare enum IceConnectionState {
+    NEW = "new",
+    CHECKING = "checking",
+    CONNECTED = "connected",
+    COMPLETED = "completed",
+    DISCONNECTED = "disconnected",
+    FAILED = "failed",
+    CLOSED = "closed"
+}

package/lib/core/data/WebRTCDataType.d.ts

@@ -0,0 +1,113 @@
+export declare enum ChannelDataType {
+    ClickData = "ClickData",
+    ClipboardData = "ClipboardData",
+    ActionCommand = "ActionCommand",
+    ActionInput = "ActionInput",
+    ActionChinese = "ActionChinese",
+    ActionRequestCloudDeviceInfo = "ActionRequestCloudDeviceInfo",
+    ActionClarity = "ActionClarity"
+}
+export declare const ActionType: Readonly<{
+    ACTION_DOWN: 0;
+    ACTION_MOVE: 2;
+    ACTION_UP: 1;
+}>;
+export type ActionType = typeof ActionType[keyof typeof ActionType];
+export declare const StreamRotation: {
+    readonly ROTATION_0: 0;
+    readonly ROTATION_90: 1;
+    readonly ROTATION_180: 2;
+    readonly ROTATION_270: 3;
+};
+export type StreamRotation = typeof StreamRotation[keyof typeof StreamRotation];
+export declare const ContainerDirection: {
+    readonly Vertical: 0;
+    readonly Horizontal: 1;
+};
+export type ContainerDirection = typeof ContainerDirection[keyof typeof ContainerDirection];
+/**
+ * TouchData - 触摸数据类，包含动作类型、指针索引、坐标、偏移时间以及触摸类型
+ */
+export declare class TouchData {
+    action: ActionType;
+    p: number;
+    x: number;
+    y: number;
+    offsetTime: number;
+    type: string;
+    constructor(action: ActionType, p: number, x: number, y: number, offsetTime: number, type?: string);
+}
+/**
+ * InputData - 文本输入数据类
+ */
+export declare class InputData {
+    text: string;
+    constructor(text: string);
+}
+/**
+ * KeyEventData - 键盘事件数据类，包含键码及修饰状态
+ */
+export declare class KeyEventData {
+    keyCode: number;
+    meta: number;
+    constructor(keyCode: number, meta: number);
+}
+export declare class ChannelData {
+    type: ChannelDataType;
+    data: string | null;
+    constructor(type: ChannelDataType, data?: string | null);
+    toString(): string;
+    /**
+     * 格式化数据
+     * 如果数据已经是字符串，则直接返回；否则使用 JSON.stringify() 转换为字符串
+     * @param data 待转换数据
+     * @returns 格式化后的字符串
+     */
+    private static formatData;
+    /**
+     * 生成点击数据
+     * @param touchData 触摸数据，可以是任意类型
+     */
+    static click(touchData: unknown): ChannelData;
+    /**
+     * 生成剪贴板数据
+     * @param data 剪贴板数据，可以是字符串或其他类型
+     */
+    static clipboard(data: unknown): ChannelData;
+    /**
+     * 生成输入数据
+     * @param data 输入数据对象
+     */
+    static input(data: unknown): ChannelData;
+    /**
+     * 生成中文输入数据
+     * @param data 中文输入数据
+     */
+    static chinese(data: unknown): ChannelData;
+    /**
+     * 生成请求云设备信息数据
+     */
+    static requestCloudDeviceInfo(): ChannelData;
+    /**
+     * 生成清晰度数据（clarity）
+     * @param data 清晰度数据
+     */
+    static clarity(data: unknown): ChannelData;
+}
+export interface RTCStatsReportExtended extends RTCStatsReport {
+    get(id: string): RTCStats | undefined;
+}
+export interface ScreenStats {
+    connectionType: string;
+    framesPerSecond: number;
+    currentRoundTripTime: number;
+    lostRate: number;
+    bitrate: string;
+    pliCount: number;
+    averageDecodeTime: number;
+}
+export interface CloudStatusPayload {
+    direction: number;
+    screenWidth: number;
+    screenHeight: number;
+}

package/lib/core/data/WebRtcError.d.ts

@@ -0,0 +1,38 @@
+export declare enum FailCode {
+    SOCKET = 10001,
+    SOCKET_CLOSE = 10002,
+    CREATE_OFFER = 10003,
+    HANDLE_OFFER = 10004,
+    CREATE_ANSWER = 10005,
+    HANDLE_ANSWER = 10006,
+    LOCAL_DES = 10007,
+    REMOTE_DES = 10008,
+    HANDLE_ICE = 10009,
+    ICE_STATE = 10010,
+    CAMERA = 10011,
+    NOT_AVAILABLE = 10012,
+    DATACHANNEL_ERR = 10013,
+    STREAM_STATE = 10014
+}
+export declare const FailInfoMap: Record<FailCode, {
+    type: string;
+    description: string;
+}>;
+export interface WebRtcError {
+    code: FailCode;
+    type: string;
+    message: string;
+    rawError?: any;
+}
+export declare function createWebRtcError(code: FailCode, rawError?: any): WebRtcError;
+export declare enum EmitType {
+    signalMessage = "signalMessage",
+    webrtcError = "webrtcError",
+    sendAnswer = "sendAnswer",
+    sendOffer = "sendOffer",
+    sendICEMessage = "sendICEMessage",
+    streamTrack = "streamTrack",
+    iceConnectionState = "iceConnectionState",
+    cloudStatusChanged = "cloudStatusChanged",
+    statisticInfo = "statisticInfo"
+}