yjz-web-sdk 1.0.9-beta.4 → 1.0.10

package/package.json

@@ -1,7 +1,7 @@
 {
   "name": "yjz-web-sdk",
   "private": false,
-  "version": "1.0.9-beta.4",
+  "version": "1.0.10",
   "type": "module",
   "description": "针对于亚矩阵项目的云手机投屏和屏幕控制",
   "license": "Apache-2.0",
@@ -20,7 +20,10 @@
   "files": [
     "package.json",
     "README.md",
-    "lib"
+    "lib",
+    "dist",
+    "src",
+    "types"
   ],
   "publishConfig": {
     "registry": "https://registry.npmjs.org",
@@ -35,8 +38,8 @@
     "@libmedia/avcodec": "1.0.1",
     "@libmedia/avformat": "1.0.1",
     "@libmedia/avnetwork": "1.0.1",
-    "@libmedia/avutil": "1.0.1",
     "@libmedia/avrender": "1.0.1",
+    "@libmedia/avutil": "1.0.1",
     "@libmedia/cheap": "~1.0.2",
     "@libmedia/common": "~2.0.1",
     "element-plus": "^2.10.5",
@@ -50,7 +53,6 @@
     "@types/node": "^22.14.1",
     "@vitejs/plugin-vue": "^6.0.1",
     "@vue/tsconfig": "^0.7.0",
-    "rollup-plugin-visualizer": "5.9.3",
     "tslib": "^2.8.1",
     "typescript": "~5.8.0",
     "vite": "^6.2.0",

package/src/assets/icon/circle.svg

@@ -0,0 +1 @@
+<svg xmlns="http://www.w3.org/2000/svg" width="24" height="24" fill="none"><circle cx="12" cy="12" r="12" fill="#fff" opacity=".6"/><circle cx="12" cy="12" r="10" fill="#000" opacity=".6"/></svg>

package/src/assets/icon/triangle.svg

@@ -0,0 +1 @@
+<svg class="icon" viewBox="0 0 1024 1024" version="1.1" xmlns="http://www.w3.org/2000/svg" width="22" height="22"><path d="M143.832313 5.834982H143.686438A108.676545 108.676545 0 0 0 5.834982 143.686438l34.499333-11.815839-34.499333 11.815839 0.072938 0.218812 0.145874 0.437624 0.583498 1.750494 2.333993 6.71023 8.752474 25.528047L49.232663 269.867929a2254749.467572 2254749.467572 0 0 1 223.917444 652.351017l9.335972 27.205605 2.552804 7.585476 0.729373 2.188119a72.572592 72.572592 0 0 0 126.181491 40.844876 72.134968 72.134968 0 0 0 14.076895-18.963693c3.282178-6.41848 5.689108-13.639271 8.023101-20.3495l0.072937-0.291749 72.572592-209.329989 47.409231-136.830334 15.53564-44.710551 0.145874-0.364687 0.510561-0.145874 45.002301-15.900327 137.486769-48.649165c99.340573-35.228705 202.984445-71.989094 209.913487-74.906584l3.355115-1.312871c8.023101-3.136303 22.391744-8.606599 33.915834-20.130689a72.499655 72.499655 0 0 0 0-102.549813L999.240712 304.877823c-1.823432-1.969307-7.293728-7.731351-13.274585-11.961714a89.056417 89.056417 0 0 0-27.205605-12.3264h-0.145874l-2.552805-0.875247L948.184617 277.161657l-27.86204-9.263034-94.672588-31.800653A405018.007245 405018.007245 0 0 1 268.919745 48.138604L178.039896 17.504947 152.657723 8.752473 145.874556 6.637292 144.196999 5.90792 143.832313 5.834982z" fill="#000000" opacity=".7"></path></svg>

package/src/components/RemotePlayer/index.vue

@@ -0,0 +1,170 @@
+<script setup lang="ts">
+import {ref, onMounted, toRefs, onBeforeUnmount} from 'vue'
+
+import useCursorStyle from '../../composables/useCursorStyle'
+import useResizeObserver from '../../composables/useResizeObserver'
+import useRemoteVideo from '../../composables/useRemoteVideo'
+import useMouseTouchControl from '../../composables/useMouseTouchControl'
+import useKeyboardControl from '../../composables/useKeyboardControl'
+import type {Dimension} from "./type";
+
+
+
+const emit = defineEmits(['channelEvent', "groupControlEvent", 'loadedSuccess', 'loadedFailure']);
+
+interface Props {
+  streamAngle?: number
+  videoAngle?: number
+  cursorType?: number
+  cloudDeviceSize?: Dimension
+  disabled?: boolean,
+  bgColor?: string,
+  isGroup?: boolean,
+}
+
+const props = withDefaults(defineProps<Props>(), {
+  streamAngle: 0,
+  videoAngle: 0,
+  cursorType: 0,
+  cloudDeviceSize: () => ({ width: 0, height: 0 }),
+  disabled: true,
+  bgColor: 'transparent',
+  isGroup: false
+})
+
+const { streamAngle, videoAngle, cursorType, cloudDeviceSize, disabled, bgColor, isGroup } = toRefs(props)
+
+const videoContainer = ref<HTMLElement | null>(null)
+const remoteVideoElement = ref<HTMLCanvasElement | null>(null)
+const audioElement = ref<HTMLAudioElement | null>(null)
+
+
+const cursorClass = useCursorStyle(cursorType)
+
+const {
+  videoSize,
+  dimensions,
+  widthRadio,
+  initVideoContainer,
+  startDecode,
+  stopPlay,
+  startPlay
+} = useRemoteVideo(videoContainer, remoteVideoElement, audioElement, videoAngle, emit)
+
+const {
+  handleMouseDown,
+  handleMouseMove,
+  handleMouseEnter,
+  handleMouseUp,
+  handleMouseLeave,
+  handleWheel
+} = useMouseTouchControl({
+  remoteVideoElement,
+  cloudDeviceSize,
+  streamAngle,
+  videoAngle,
+  widthRadio,
+  isGroup,
+  emit
+})
+
+const {
+  startListening,
+  stopListening
+} = useKeyboardControl(disabled, emit)
+
+useResizeObserver(videoContainer, dimensions)
+
+const handleClick = () => {
+  if (audioElement.value) {
+    audioElement.value.muted = false;
+    audioElement.value.play().catch(()=>{});
+  }
+};
+
+onMounted(() => {
+  document.addEventListener("click", handleClick);
+  initVideoContainer()
+})
+
+
+
+onBeforeUnmount(() => {
+  document.removeEventListener("click", handleClick);
+})
+
+defineExpose({ startDecode, stopPlay, remoteVideoElement, startPlay })
+</script>
+
+<template>
+  <div ref="videoContainer" class="flex flex-1 items-center justify-center" style="width:100%;height: 100%;position:relative;overflow:hidden">
+    <div ref="keyboardArea" @pointerenter="startListening" @pointerleave="stopListening"
+         class="vContainer"
+         :style="{height: videoSize.height + 'px', width: videoSize.width + 'px', transform: `rotate(${videoAngle}deg)`}" style="position: relative;overflow:hidden">
+      <canvas ref="remoteVideoElement"
+             class="canvas-control"
+              :width="cloudDeviceSize.width"
+              :height="cloudDeviceSize.height"
+             :class="[cursorClass, { 'no-events': disabled }]"
+             @pointerenter="handleMouseEnter"
+             @pointerdown="handleMouseDown"
+             @pointermove="handleMouseMove"
+             @pointerup="handleMouseUp"
+             @pointerleave="handleMouseLeave"
+             @wheel="handleWheel"
+              hidden="hidden"
+             :style="{backgroundColor: `${bgColor}`}"></canvas>
+      <audio ref="audioElement" autoplay playsinline muted style="display: none"></audio>
+    </div>
+  </div>
+</template>
+
+<style scoped>
+.vContainer {
+  transition: transform 0.2s linear;
+  transform-origin: center center;
+  margin: 0;       /* 清除外边距 */
+  overflow: hidden;
+  /* 对频繁旋转或缩放使用 translateZ(0) 强制 GPU 合成 */
+  transform: translateZ(0);
+  will-change: transform; /* 告诉浏览器只关注 transform 变化 */
+}
+.flex {
+  display: flex;
+}
+.flex-1 {
+  flex: 1 1 0;
+}
+.items-center {
+  align-items: center;
+}
+.justify-center {
+  justify-content: center;
+}
+.canvas-control {
+  width: 100%;
+  height: 100%;
+  display: block;
+  user-select: none;
+  -webkit-user-drag: none;
+  touch-action: none;
+
+  object-fit: cover;
+
+  backface-visibility: hidden;
+  transform: translateZ(0);
+  clip-path: inset(0 1px);
+}
+.circle-cursor {
+  cursor: url('../../assets/icon/circle.svg') 12 12, auto;
+}
+.triangle-cursor {
+  cursor: url('../../assets/icon/triangle.svg') 1 1, auto;
+}
+.default-cursor {
+  cursor: default;
+}
+.no-events {
+  pointer-events: none !important;
+}
+</style>

package/src/components/RemotePlayer/type.ts

@@ -0,0 +1,11 @@
+export interface Dimension {
+    width: number;
+    height: number;
+}
+
+export interface TouchEventData{
+    clientX: number,
+    clientY: number,
+    timeStamp: DOMHighResTimeStamp,
+}
+

package/src/composables/useCursorStyle.ts

@@ -0,0 +1,15 @@
+import type {Ref} from 'vue'
+import {computed} from 'vue'
+
+export default function useCursorStyle(cursorType: Ref<number>) {
+    return computed(() => {
+        switch (cursorType.value) {
+            case 1:
+                return 'circle-cursor'
+            case 2:
+                return 'triangle-cursor'
+            default:
+                return 'default-cursor'
+        }
+    })
+}

package/src/composables/useKeyboardControl.ts

@@ -0,0 +1,32 @@
+import {ChannelDataType, KeyEventData} from "../core/data/WebrtcDataType";
+import {getKeyEventData} from "../core/util/KeyCodeUtil";
+import {type Ref, ref} from "vue";
+
+
+export default function useKeyboardControl(
+    disabled:Ref<boolean>,
+    emit: (event: 'channelEvent', payload: ChannelDataType, data:any) => void
+) {
+    const listening = ref(false)
+    const handleKeyDown = (e: KeyboardEvent) => {
+        const data: KeyEventData = getKeyEventData(e)
+        emit('channelEvent', ChannelDataType.ActionInput, data)
+    }
+
+    const startListening = () => {
+        if (!disabled){return}
+        listening.value = true;
+        document.addEventListener('keydown', handleKeyDown);
+    }
+
+    const stopListening = () => {
+        if (!disabled){return}
+        listening.value = false;
+        document.removeEventListener('keydown', handleKeyDown);
+    }
+
+    return {
+        startListening,
+        stopListening
+    }
+}

package/src/composables/useMouseTouchControl.ts

@@ -0,0 +1,158 @@
+import type {Ref} from 'vue'
+import {ref} from 'vue'
+import {ActionType, ChannelDataType, TouchData, WheelData} from "../core/data/WebrtcDataType";
+import {transformCoordinate, valueToPercentage} from "../core/util/ScreenControlUtil";
+import type {Dimension, TouchEventData} from "../components/RemotePlayer/type";
+
+interface MouseTouchControlOptions {
+    remoteVideoElement: Ref<HTMLCanvasElement | null>
+    cloudDeviceSize:Ref<Dimension>
+    streamAngle: Ref<number>
+    videoAngle: Ref<number>
+    widthRadio: Ref<number>
+    isGroup: Ref<boolean>
+    emit: (event: 'channelEvent', payload: ChannelDataType, data: any) => void
+}
+
+export default function useMouseTouchControl(options: MouseTouchControlOptions) {
+    const {
+        remoteVideoElement,
+        cloudDeviceSize,
+        streamAngle,
+        videoAngle,
+        widthRadio,
+        isGroup,
+        emit
+    } = options
+
+    const isPointerDown = ref(false)
+    const bound = ref(0)
+    const pointerList = ref<number[]>(new Array(20).fill(0));
+    const pointerDownTime = ref<number[]>(new Array(10).fill(0));
+    const index = ref<number>(0)
+    const groupIndex = ref<number>(0)
+
+    const handlePointerEvent = (event: PointerEvent | TouchEventData, action: ActionType) => {
+        if (!remoteVideoElement.value) return;
+        const offsetTime = Math.trunc((event.timeStamp - pointerDownTime.value[0]));
+        const rect = remoteVideoElement.value.getBoundingClientRect();
+
+        let x = event.clientX - rect.left;
+        let y = event.clientY - rect.top;
+
+        const cloudWidth = cloudDeviceSize.value.width;
+        const cloudHeight = cloudDeviceSize.value.height;
+        const result = transformCoordinate(
+            rect.width, rect.height, cloudWidth, cloudHeight,
+            videoAngle.value, streamAngle.value, x, y
+        );
+        if (!result || result.length < 2) return;
+        x = result[0];
+        y = result[1];
+
+        if (action === ActionType.ACTION_MOVE) {
+            const flipY = pointerList.value[10] - y;
+            const flipX = pointerList.value[0] - x;
+            if (Math.abs(flipY) < bound.value && Math.abs(flipX) < bound.value) return;
+        }
+        if (isGroup.value) {
+            if(action === ActionType.ACTION_DOWN){
+                index.value = 0
+                groupIndex.value ++
+            }else if(action === ActionType.ACTION_UP || action === ActionType.ACTION_MOVE){
+                index.value ++
+            }
+        }
+        // 更新最后一次位置
+        pointerList.value[0] = x
+        pointerList.value[10] = y
+
+        // 转换为百分比坐标
+        const [px, py] = valueToPercentage(
+            rect.width, rect.height, cloudWidth, cloudHeight,
+            videoAngle.value, streamAngle.value, x, y
+        )
+
+        // 构造发送数据
+        const touchData = isGroup.value
+            ? new TouchData(action, 0, px, py, offsetTime, "web", index.value, groupIndex.value)
+            : new TouchData(action, 0, px, py, offsetTime, "web")
+
+        emit('channelEvent', ChannelDataType.ClickData, touchData)
+
+    }
+
+    const handleMouseDown = (event: PointerEvent) => {
+        // 标记按下状态
+        isPointerDown.value = true
+        if(remoteVideoElement.value){
+            remoteVideoElement.value.setPointerCapture(event.pointerId)
+        }
+        pointerDownTime.value[0] = event.timeStamp;
+        bound.value = Math.trunc(6 / widthRadio.value);
+        // 群控模式下新一轮点击重置 index
+        handlePointerEvent(event, ActionType.ACTION_DOWN)
+    }
+
+    const handleMouseMove = (event: PointerEvent) => {
+        if (!isPointerDown.value) return;
+        handlePointerEvent(event, ActionType.ACTION_MOVE)
+        if(!remoteVideoElement.value){
+            return;
+        }
+        const rect = remoteVideoElement.value.getBoundingClientRect();
+        const { clientX: x, clientY: y } = event;
+        if (x < rect.left || x > rect.right || y < rect.top || y > rect.bottom) {
+            handleMouseUp(event)
+        }
+    }
+
+    const handleMouseUp = (event: PointerEvent) => {
+        if (!isPointerDown.value) return;
+        isPointerDown.value = false
+        handlePointerEvent(event, ActionType.ACTION_UP)
+        // 释放指针捕获 & 清除长按
+        if(remoteVideoElement.value){
+            remoteVideoElement.value.releasePointerCapture(event.pointerId)
+        }
+    }
+
+    const handleMouseEnter = (event: PointerEvent) => {
+        // 如果左键已按下且尚未标记为 down，就补发一次 down
+        if (event.buttons === 1 && !isPointerDown.value) {
+            isPointerDown.value = true
+            if(remoteVideoElement.value){
+                remoteVideoElement.value.setPointerCapture(event.pointerId)
+            }
+            pointerDownTime.value[0] = event.timeStamp;
+            bound.value = Math.trunc(6 / widthRadio.value);
+
+            handlePointerEvent(event, ActionType.ACTION_DOWN)
+        }
+    }
+    const handleMouseLeave = (event: PointerEvent) => {
+        if (isPointerDown.value) {
+            handlePointerEvent(event, ActionType.ACTION_UP);
+            isPointerDown.value = false
+            if(remoteVideoElement.value){
+                remoteVideoElement.value.releasePointerCapture(event.pointerId)
+            }
+        }
+    }
+
+    const handleWheel = (e: WheelEvent) => {
+        const delta = Math.sign(e.deltaY) * -1;
+        const wheelData = new WheelData(delta);
+        emit('channelEvent', ChannelDataType.ActionWheel,wheelData)
+    }
+
+    return {
+        isPointerDown,
+        handleMouseDown,
+        handleMouseMove,
+        handleMouseEnter,
+        handleMouseUp,
+        handleMouseLeave,
+        handleWheel
+    }
+}

package/src/composables/useRemoteVideo.ts

@@ -0,0 +1,248 @@
+import type {Ref} from 'vue';
+import {computed, onBeforeUnmount, onMounted, ref} from 'vue';
+import {Dimension} from '../components/RemotePlayer/type';
+import {
+    checkDecoderSupport,
+    detectRenderingCapabilities,
+    VideoCodecType
+} from "../util/WasmUtil";
+import DecodeWorker from '../worker/worker.ts?worker&inline';
+
+import {DecoderSupportResult, RendererType} from "../types";
+
+export default function useRemoteVideo(
+    videoContainer: Ref<HTMLElement | null>,
+    remoteVideoElement: Ref<HTMLCanvasElement | null>,
+    audioElement: Ref<HTMLAudioElement | null>,
+    videoAngle: Ref<number>,
+    emit: (event: 'loadedSuccess' | 'loadedFailure', reason?: string) => void
+) {
+    const screenStatus = ref(false);
+    const remoteVideo = ref<Partial<Dimension>>({});
+    const dimensions = ref<Dimension>({ width: 0, height: 0 });
+    const widthRadio = ref(0);
+    const heightRadio = ref(0);
+    const resizeObserver = ref<ResizeObserver | null>(null);
+
+    const renderType = ref<RendererType | null>(null)
+
+    const isRotated = computed(() => videoAngle.value % 180 !== 0);
+    // Worker 相关
+    const worker = ref<Worker | null>(null)
+    const offscreen =  ref<OffscreenCanvas | null>(null)
+    const workerInited = ref(false)
+    const visible = ref<boolean>(true);
+
+    const videoSize = computed(() => {
+        let width: number;
+        let height: number;
+
+        const containerWidth = isRotated.value ? dimensions.value.height : dimensions.value.width;
+        const containerHeight = isRotated.value ? dimensions.value.width : dimensions.value.height;
+
+        const vWidth = remoteVideo.value.width ?? 720;
+        const vHeight = remoteVideo.value.height ?? 1280;
+        const aspect = vWidth / vHeight;
+        const videoHeight = containerWidth / aspect;
+        if (videoHeight > containerHeight) {
+            height = containerHeight;
+            width = height * aspect;
+        } else {
+            width = containerWidth;
+            height = videoHeight;
+        }
+
+        widthRadio.value = vWidth / width;
+        heightRadio.value = vHeight / height;
+
+        return { width, height };
+    });
+    const waitingKeyFrame = ref(true)
+    const checkResult = ref<DecoderSupportResult | null>(null)
+
+    const observeContainer = () => {
+        if (!videoContainer.value) return;
+        resizeObserver.value = new ResizeObserver(([entry]) => {
+            const { width, height } = entry.contentRect;
+            dimensions.value = { width, height };
+        });
+        resizeObserver.value.observe(videoContainer.value);
+    };
+
+    const initVideoContainer = () => {
+        observeContainer();
+        bindVideoEvents()
+    };
+
+    const bindVideoEvents = async () => {
+        const video = remoteVideoElement.value;
+        if (!video) return;
+        checkResult.value = await checkDecoderSupport(VideoCodecType.H264)
+        console.log("checkResult.value====>", checkResult.value)
+        const result = await detectRenderingCapabilities()
+        console.log("detectRenderingCapabilities===>", result)
+        renderType.value = result.best
+        initWorker(video)
+    };
+
+    const initWorker = (video: HTMLCanvasElement) => {
+        if (workerInited.value) return;
+        console.log("initWorker===>");
+        worker.value = new DecodeWorker();
+        const pixelRatio = window.devicePixelRatio || 1;
+        if (!offscreen.value) {
+            offscreen.value = video.transferControlToOffscreen();
+        }
+
+        worker.value.onmessage = (event: MessageEvent) => {
+            const { data } = event;
+            switch (data.type) {
+                case 'decoderError':
+                case 'rendererError':
+                    emit('loadedFailure', data.error)
+                    break
+            }
+        }
+        worker.value.onerror = (event: ErrorEvent) => {
+            console.error('error', event);
+        }
+
+        worker.value?.postMessage({
+            type: 'init',
+            supportH265: checkResult.value?.supported,
+            isHardware: checkResult.value?.hardware,
+            isSoftware: checkResult.value?.software,
+            canvas: offscreen.value,
+            pixelRatio: pixelRatio,
+            renderType: renderType.value
+        }, [offscreen.value])
+        workerInited.value = true;
+    }
+
+    const startDecode = (arrayBuffer: ArrayBuffer) => {
+        handleMessage(arrayBuffer)
+    };
+
+    const handleMessage = (arrayBuffer: ArrayBuffer) => {
+        const data = new Uint8Array(arrayBuffer); // ArrayBuffer -> Uint8Array
+        if (data.length < 4) return;
+
+        // 读取 label 长度（前 4 个字节，Big Endian）
+        const labelLen = (data[0] << 24) | (data[1] << 16) | (data[2] << 8) | data[3];
+        if (data.length < 4 + labelLen) return;
+
+        // 读取 label
+        const labelBytes = data.subarray(4, 4 + labelLen);
+        const label = new TextDecoder().decode(labelBytes);
+
+        // 读取 frame
+        const frame = data.subarray(4 + labelLen); // Uint8Array 视图，底层 buffer 没拷贝
+
+        // 调用回调
+        decodeVideo(label, frame)
+    }
+
+    const decodeVideo = (label: String, data: Uint8Array) => {
+        if(label !== 'key' && !visible){
+            return;
+        }
+        if (label === 'config'){
+            return;
+        }
+        if (waitingKeyFrame.value){
+            if(label === 'delta'){
+                return;
+            }
+            waitingKeyFrame.value = false;
+            emit("loadedSuccess")
+        }
+        worker.value?.postMessage(
+            {
+                type: 'decode',
+                label,
+                data
+            },
+            [data.buffer]
+        );
+    }
+
+    const stopPlay = () => {
+        screenStatus.value = false;
+        waitingKeyFrame.value = true;
+        const audio = audioElement.value;
+        worker.value?.postMessage({type: 'clearRender'});
+        if (audio && audio.srcObject) {
+            const stream = audio.srcObject as MediaStream;
+
+            stream.getTracks().forEach(t => t.stop());
+
+            audio.pause();
+            audio.srcObject = null;
+        }
+    };
+
+    const startPlay = (track: MediaStreamTrack) => {
+        const audio = audioElement.value;
+        if (!audio) return;
+
+        replaceAudioTrack(audio, track);
+        audio.autoplay = true;
+        audio.muted = false;
+        audio.setAttribute('playsinline', 'true');
+        audio.setAttribute('webkit-playsinline', 'true');
+    };
+
+    const replaceAudioTrack = (audioEl: HTMLAudioElement, newTrack: MediaStreamTrack) => {
+        if (!audioEl.srcObject) {
+            audioEl.srcObject = new MediaStream();
+        }
+
+        const stream = audioEl.srcObject as MediaStream;
+        updateTrack(stream, newTrack);
+    };
+
+    const updateTrack = (stream: MediaStream, newTrack: MediaStreamTrack) => {
+        if (newTrack.kind === 'audio') {
+            // 移除旧音频轨
+            stream.getAudioTracks().forEach(oldTrack => {
+                stream.removeTrack(oldTrack);
+                oldTrack.stop();
+            });
+            stream.addTrack(newTrack);
+        }
+    };
+
+    const handleVisibilityChange = () => {
+        visible.value = document.visibilityState === "visible";
+    };
+
+    onMounted(() => {
+        document.addEventListener('visibilitychange', handleVisibilityChange);
+    })
+
+    onBeforeUnmount(async () => {
+        document.removeEventListener('visibilitychange', handleVisibilityChange);
+        if (resizeObserver.value && videoContainer.value) {
+            resizeObserver.value.unobserve(videoContainer.value);
+        }
+        stopPlay();
+        worker.value?.postMessage({type: 'stopDecode'});
+        if(worker.value) {
+            worker.value.terminate()
+        }
+    });
+
+
+    return {
+        videoSize,
+        remoteVideo,
+        dimensions,
+        widthRadio,
+        heightRadio,
+        screenStatus,
+        initVideoContainer,
+        startDecode,
+        stopPlay,
+        startPlay
+    };
+}