qt-human 2.2.0-alpha.9 → 2.3.1-alpha

package/dist/index.d.ts

@@ -10,6 +10,8 @@ import { ThreeJSON } from '@/types/model';
 import { QuanTe as QuanTe$2 } from '@/types';
 import Recorder from 'recorder-core';
 import { IResultData as IResultData$1 } from '@/api/interface';
+import FramePlayer from '@/utils/framePlayer';
+import { CameraConfig as CameraConfig$1 } from '@/types/camera';
 
 type ActionObjType = {
     action: THREE.AnimationAction;
@@ -70,7 +72,8 @@ declare class ThreeJSApp extends Emittery {
     gui: SceneGUI | null;
     bgScene: THREE.Scene | null;
     bgCamera: THREE.OrthographicCamera | null;
-    constructor(container: (string | HTMLElement) | undefined, config: QuanTe$1.Model.Description);
+    private humanObj;
+    constructor(container: (string | HTMLElement) | undefined, config: QuanTe$1.Model.Description, humanObj: Human);
     init(): void;
     setContainer(): void;
     updateLight(): void;
@@ -107,6 +110,7 @@ declare class ThreeJSApp extends Emittery {
     startRender(): void;
     stopRender(): void;
     gsapMorph(target: number[], source: number[], t2: any): void;
+    private getDefaultAniCode;
     initIdle: lodash.DebouncedFunc<() => void>;
     playAction: lodash.DebouncedFuncLeading<(code: string, opts?: QuanTe$1.PlayActionOptions) => void>;
     stopAllActionExcludeIdle(): void;
@@ -269,13 +273,38 @@ declare class VoiceInteraction extends Emittery {
     private wakeWordDetection;
     private exitWordDetection;
     private interruptWordDetection;
-    private onWakeDetected;
-    private onSleepDetected;
+    onWakeDetected(): void;
+    onSleepDetected(): void;
     private onInterruptDetected;
     destroy(): void;
 }
 declare const _default: VoiceInteraction;
 
+type FacingMode = "user" | "environment";
+
+interface CameraConfig {
+  el: string | HTMLDivElement;
+  facingMode?: FacingMode;       // 前/后摄像头
+  features?: string[];           // 功能列表 ["photo", "detection"]
+  overlayImage?: HTMLImageElement | null; // 合影 PNG
+}
+
+type ProximityState = 'near' | 'far' | 'stable';
+
+declare class CameraSDK {
+    private manager;
+    private features;
+    private container;
+    private video;
+    constructor(config: CameraConfig);
+    open(): Promise<void>;
+    close(): void;
+    takePhoto(): Promise<string>;
+    switchCamera(facingMode: "user" | "environment"): Promise<void>;
+    detectPose(callback: (stable: ProximityState) => void): Promise<void>;
+    destroy(): void;
+}
+
 declare class Human extends Emittery implements QuanTe$1.Human {
     name: string;
     core: ThreeJSApp | null;
@@ -297,8 +326,9 @@ declare class Human extends Emittery implements QuanTe$1.Human {
     private isNotRender;
     private mode;
     private ttsclient;
-    private framePlayer;
+    framePlayer: FramePlayer | null;
     voiceInter: typeof _default;
+    camera: CameraSDK | null;
     constructor(config: QuanTe$1.Configuration);
     private refreshGatewayToken;
     private initTTSClient;
@@ -344,6 +374,8 @@ declare class Human extends Emittery implements QuanTe$1.Human {
     stopRender(): Promise<boolean>;
     closeRecorder(): Promise<boolean>;
     openRecorder(): Promise<QuanTe$1.Result<QuanTe$1.Recorder.PermissionResult>>;
+    openCamera(config: CameraConfig$1): Promise<boolean>;
+    wakeUp(isWake?: boolean): Promise<void>;
 }
 
 declare namespace QuanTe {
@@ -454,6 +486,8 @@ declare namespace QuanTe {
     public openVoiceInteraction(config: VoiceConfig): Promise<boolean>
     public closeRecorder(): Promise<boolean>
     public openRecorder(): Promise<QuanTe.Result<QuanTe.Recorder.PermissionResult>>
+    // 打开摄像头
+    public openCamera(config: CameraConfig): Promise<boolean>
   }
 
   export interface ModelInfo {
@@ -1039,6 +1073,8 @@ declare enum EmitEvent {
   AUDIO_PAUSE = 'audio.pause',
   // 播放音频
   AUDIO_PLAY = 'audio.play',
+  // 摄像机人体检测结果
+  CAMERA_PERSON_TEST_RESULT = 'camera.person.test.result'
 }
 
 export { EmitEvent, QuanTe, Human as default };

package/dist/types/modules/camera/feature.d.ts

@@ -0,0 +1 @@
+export declare const FeatureRegistry: Record<string, any>;

package/dist/types/modules/camera/index.d.ts

@@ -0,0 +1,15 @@
+import type { CameraConfig } from "../../types/camera";
+import { ProximityState } from './proximityDetector';
+export declare class CameraSDK {
+    private manager;
+    private features;
+    private container;
+    private video;
+    constructor(config: CameraConfig);
+    open(): Promise<void>;
+    close(): void;
+    takePhoto(): Promise<string>;
+    switchCamera(facingMode: "user" | "environment"): Promise<void>;
+    detectPose(callback: (stable: ProximityState) => void): Promise<void>;
+    destroy(): void;
+}

package/dist/types/modules/camera/mediaStream.d.ts

@@ -0,0 +1,12 @@
+import type { FacingMode, CameraCore } from "../../types/camera";
+export declare class MediaStreamManager implements CameraCore {
+    video: HTMLVideoElement;
+    private stream;
+    private facingMode;
+    constructor(video: HTMLVideoElement, facingMode?: FacingMode);
+    open(): Promise<void>;
+    close(): void;
+    switch(facingMode: FacingMode): Promise<void>;
+    getVideoEl(): HTMLVideoElement;
+    getStream(): MediaStream | null;
+}

package/dist/types/modules/camera/photoFeature.d.ts

@@ -0,0 +1,7 @@
+import type { CameraFeature, CameraCore } from "../../types/camera";
+export declare class PhotoFeature implements CameraFeature {
+    name: string;
+    private overlay;
+    constructor(overlay?: HTMLImageElement | null);
+    takePhoto(camera: CameraCore): Promise<string>;
+}

package/dist/types/modules/camera/poseDetector.d.ts

@@ -0,0 +1,15 @@
+import * as posedetection from "@tensorflow-models/pose-detection";
+import "@tensorflow/tfjs-backend-webgl";
+import { MediaStreamManager } from "./mediaStream";
+import { ProximityState } from "./proximityDetector";
+export declare class PoseDetector {
+    private detector;
+    private mediaStrea;
+    private interval;
+    private proximityDetector;
+    constructor();
+    init(mediaStrea: MediaStreamManager): Promise<void>;
+    on(callback: (stable: ProximityState) => void): void;
+    detect(video: HTMLVideoElement): Promise<posedetection.Pose[]>;
+    destroy(): Promise<void>;
+}

package/dist/types/modules/camera/proximityDetector.d.ts

@@ -0,0 +1,11 @@
+import { Keypoint } from '@tensorflow-models/pose-detection';
+export type ProximityState = 'near' | 'far' | 'stable';
+export declare class ProximityDetector {
+    private baselineWidth;
+    private lastState;
+    private threshold;
+    private callback?;
+    constructor(threshold?: number);
+    onChange(callback: (state: ProximityState) => void): void;
+    update(keypoints: Keypoint[]): void;
+}

package/dist/types/modules/core/render.d.ts

@@ -6,6 +6,7 @@ import Emittery from 'emittery';
 import type { QuanTe } from '@/types/index';
 import { SceneGUI } from '@/utils/sceneGUI';
 import type { ThreeJSON } from '@/types/model';
+import Human from '../human';
 type ActionObjType = {
     action: THREE.AnimationAction;
     name: string;
@@ -65,7 +66,8 @@ declare class ThreeJSApp extends Emittery {
     gui: SceneGUI | null;
     bgScene: THREE.Scene | null;
     bgCamera: THREE.OrthographicCamera | null;
-    constructor(container: (string | HTMLElement) | undefined, config: QuanTe.Model.Description);
+    private humanObj;
+    constructor(container: (string | HTMLElement) | undefined, config: QuanTe.Model.Description, humanObj: Human);
     init(): void;
     setContainer(): void;
     updateLight(): void;
@@ -102,6 +104,7 @@ declare class ThreeJSApp extends Emittery {
     startRender(): void;
     stopRender(): void;
     gsapMorph(target: number[], source: number[], t2: any): void;
+    private getDefaultAniCode;
     initIdle: import("lodash").DebouncedFunc<() => void>;
     playAction: import("lodash").DebouncedFuncLeading<(code: string, opts?: QuanTe.PlayActionOptions) => void>;
     stopAllActionExcludeIdle(): void;

package/dist/types/modules/core/voice-interaction.d.ts

@@ -37,8 +37,8 @@ declare class VoiceInteraction extends Emittery {
     private wakeWordDetection;
     private exitWordDetection;
     private interruptWordDetection;
-    private onWakeDetected;
-    private onSleepDetected;
+    onWakeDetected(): void;
+    onSleepDetected(): void;
     private onInterruptDetected;
     destroy(): void;
 }

package/dist/types/modules/human.d.ts

@@ -5,6 +5,9 @@ import Recorder from './core/recorder';
 import type { QuanTe } from '@/types/index';
 import { IResultData } from '@/api/interface';
 import voiceInteraction from './core/voice-interaction';
+import FramePlayer from '@/utils/framePlayer';
+import { CameraConfig } from '@/types/camera';
+import { CameraSDK } from './camera';
 declare class Human extends Emittery implements QuanTe.Human {
     name: string;
     core: ThreeJSApp | null;
@@ -26,8 +29,9 @@ declare class Human extends Emittery implements QuanTe.Human {
     private isNotRender;
     private mode;
     private ttsclient;
-    private framePlayer;
+    framePlayer: FramePlayer | null;
     voiceInter: typeof voiceInteraction;
+    camera: CameraSDK | null;
     constructor(config: QuanTe.Configuration);
     private refreshGatewayToken;
     private initTTSClient;
@@ -73,6 +77,8 @@ declare class Human extends Emittery implements QuanTe.Human {
     stopRender(): Promise<boolean>;
     closeRecorder(): Promise<boolean>;
     openRecorder(): Promise<QuanTe.Result<QuanTe.Recorder.PermissionResult>>;
+    openCamera(config: CameraConfig): Promise<boolean>;
+    wakeUp(isWake?: boolean): Promise<void>;
 }
 export default Human;
 export type { QuanTe };

package/dist/types/types/camera.d.ts

@@ -0,0 +1,19 @@
+export type FacingMode = "user" | "environment";
+
+export interface CameraConfig {
+  el: string | HTMLDivElement;
+  facingMode?: FacingMode;       // 前/后摄像头
+  features?: string[];           // 功能列表 ["photo", "detection"]
+  overlayImage?: HTMLImageElement | null; // 合影 PNG
+}
+
+export interface CameraFeature {
+  name: string;
+  init?(camera: CameraCore): Promise<void> | void;
+  destroy?(): Promise<void> | void;
+}
+
+export interface CameraCore {
+  getVideoEl(): HTMLVideoElement;
+  getStream(): MediaStream | null;
+}

package/dist/types/types/enum.d.ts

@@ -16,5 +16,6 @@ export declare enum EmitEvent {
     WAKE_RECORDER_START = "wake.record.start",
     WAKE_RECORDER_STOP = "wake.record.stop",
     AUDIO_PAUSE = "audio.pause",
-    AUDIO_PLAY = "audio.play"
+    AUDIO_PLAY = "audio.play",
+    CAMERA_PERSON_TEST_RESULT = "camera.person.test.result"
 }

package/dist/types/types/enum.ts

@@ -35,4 +35,6 @@ export enum EmitEvent {
   AUDIO_PAUSE = 'audio.pause',
   // 播放音频
   AUDIO_PLAY = 'audio.play',
+  // 摄像机人体检测结果
+  CAMERA_PERSON_TEST_RESULT = 'camera.person.test.result'
 }

package/dist/types/types/index.d.ts

@@ -1,6 +1,7 @@
 import { API } from "./api";
 import * as THREE from 'three'
 import { GLTF } from 'three/examples/jsm/loaders/GLTFLoader.js';
+import { CameraConfig } from "./camera";
 
 export namespace QuanTe {
   export type Character = 'xiaomeng' | 'custom' | 'quan' | 'doctor' | 'boy' | 'girl' | '';
@@ -110,6 +111,8 @@ export namespace QuanTe {
     public openVoiceInteraction(config: VoiceConfig): Promise<boolean>
     public closeRecorder(): Promise<boolean>
     public openRecorder(): Promise<QuanTe.Result<QuanTe.Recorder.PermissionResult>>
+    // 打开摄像头
+    public openCamera(config: CameraConfig): Promise<boolean>
   }
 
   export interface ModelInfo {

package/dist/types/utils/framePlayer.d.ts

@@ -24,5 +24,6 @@ export default class FramePlayer extends Emittery {
     getPlayInfo(): QuanTe.API.StreamAudioPlayInfo;
     setAudioRate(rate: number): Promise<void>;
     private mergeArrayBuffers;
+    getPlayingStatus(): boolean;
 }
 export {};

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "qt-human",
-  "version": "2.2.0-alpha.9",
+  "version": "2.3.1-alpha",
   "main": "dist/bundle.umd.js",
   "module": "dist/bundle.esm.js",
   "types": "dist/types/index.d.ts",
@@ -57,6 +57,13 @@
     "web-streams-polyfill": "^4.0.0"
   },
   "dependencies": {
+    "@mediapipe/pose": "^0.5.1675469404",
+    "@tensorflow-models/pose-detection": "^2.1.3",
+    "@tensorflow/tfjs-backend-cpu": "^4.22.0",
+    "@tensorflow/tfjs-backend-webgl": "^4.22.0",
+    "@tensorflow/tfjs-backend-webgpu": "^4.22.0",
+    "@tensorflow/tfjs-converter": "^4.22.0",
+    "@tensorflow/tfjs-core": "^4.22.0",
     "axios": "^1.7.2",
     "crypto-js": "^4.2.0",
     "emittery": "^1.0.3",