qt-human 2.3.5-alpha → 2.3.6-alpha

package/dist/index.d.ts

@@ -11,7 +11,7 @@ import { QuanTe as QuanTe$2 } from '@/types';
 import Recorder from 'recorder-core';
 import { IResultData as IResultData$1 } from '@/api/interface';
 import FramePlayer from '@/utils/framePlayer';
-import { CameraConfig as CameraConfig$1 } from '@/types/camera';
+import { CameraConfig as CameraConfig$1, FaceRecognitionConfig as FaceRecognitionConfig$1 } from '@/types/camera';
 
 type ActionObjType = {
     action: THREE.AnimationAction;
@@ -273,7 +273,7 @@ declare class VoiceInteraction extends Emittery {
     private wakeWordDetection;
     private exitWordDetection;
     private interruptWordDetection;
-    onWakeDetected(): void;
+    onWakeDetected(echo?: any): void;
     onSleepDetected(): void;
     private onInterruptDetected;
     destroy(): void;
@@ -289,19 +289,35 @@ interface CameraConfig {
   overlayImage?: HTMLImageElement | null; // 合影 PNG
 }
 
-type ProximityState = 'near' | 'far' | 'stable';
+interface FaceRecognitionConfig {
+  // 多长时间识别一次 单位 ms, 默认 3000ms
+  duration?: number;
+  // 识别人数 默认 不限
+  maxResults?: number;
+  // 是否在视频上绘制人脸检测框，默认 false
+  drawDetections?: boolean;
+  // 是否在视频上绘制人脸特征点，默认 false
+  drawFaceLandmarks?: boolean;
+  // 最小置信度，默认 0.6
+  minConfidence?: number; 
+}
 
+type ProximityState = 'near' | 'far' | 'stable';
 declare class CameraSDK {
     private manager;
     private features;
     private container;
     private video;
     constructor(config: CameraConfig);
-    open(): Promise<void>;
+    open(): Promise<boolean>;
+    isOpen(): boolean;
     close(): void;
     takePhoto(): Promise<string>;
     switchCamera(facingMode: "user" | "environment"): Promise<void>;
-    detectPose(callback: (stable: ProximityState) => void): Promise<void>;
+    startFaceRecognition(config: FaceRecognitionConfig, callback: Function): Promise<boolean>;
+    stopFaceRecognition(): Promise<boolean>;
+    startBodyDetect(callback: (stable: ProximityState) => void): Promise<boolean>;
+    stopBodyDetect(): Promise<boolean>;
     destroy(): void;
 }
 
@@ -375,7 +391,11 @@ declare class Human extends Emittery implements QuanTe$1.Human {
     closeRecorder(): Promise<boolean>;
     openRecorder(): Promise<QuanTe$1.Result<QuanTe$1.Recorder.PermissionResult>>;
     openCamera(config: CameraConfig$1): Promise<boolean>;
-    wakeUp(isWake?: boolean): Promise<void>;
+    wakeUp(isWake: boolean | undefined, echo: any): Promise<void>;
+    startFaceRecognition(config?: FaceRecognitionConfig$1): Promise<boolean>;
+    stopFaceRecognition(): Promise<boolean>;
+    startBodyDetect(): Promise<boolean>;
+    stopBodyDetect(): Promise<boolean>;
 }
 
 declare namespace QuanTe {
@@ -488,6 +508,10 @@ declare namespace QuanTe {
     public openRecorder(): Promise<QuanTe.Result<QuanTe.Recorder.PermissionResult>>
     // 打开摄像头
     public openCamera(config: CameraConfig): Promise<boolean>
+    public startFaceRecognition(config: FaceRecognitionConfig): Promise<boolean>
+    public stopFaceRecognition(): Promise<boolean>
+    public startBodyDetect(): Promise<boolean>
+    public stopBodyDetect(): Promise<boolean>
   }
 
   export interface ModelInfo {
@@ -1074,7 +1098,9 @@ declare enum EmitEvent {
   // 播放音频
   AUDIO_PLAY = 'audio.play',
   // 摄像机人体检测结果
-  CAMERA_PERSON_TEST_RESULT = 'camera.person.test.result'
+  CAMERA_PERSON_TEST_RESULT = 'camera.person.test.result',
+  // 摄像机人脸识别结果
+  CAMERA_FACE_RECOGNITION_RESULT = 'camera.face.recognition.result'
 }
 
 export { EmitEvent, QuanTe, Human as default };

package/dist/types/api/face.d.ts

@@ -0,0 +1,2 @@
+import { IResultData } from './interface';
+export declare const faceRecognitionApi: (blob: Blob) => Promise<IResultData<any>>;

package/dist/types/modules/camera/faceFeature.d.ts

@@ -0,0 +1,13 @@
+import type { CameraFeature, CameraCore, FaceRecognitionConfig } from "../../types/camera";
+export declare class FaceFeature implements CameraFeature {
+    name: string;
+    isInitialized: boolean;
+    private interval;
+    constructor();
+    initFaceAPI(): Promise<void>;
+    startFaceRecognition(camera: CameraCore, config: FaceRecognitionConfig, callback: Function): Promise<boolean>;
+    private expandBox;
+    private cropFaceToBase64;
+    private cropFaceToBlob;
+    stopFaceRecognition(): Promise<boolean>;
+}

package/dist/types/modules/camera/index.d.ts

@@ -1,15 +1,19 @@
-import type { CameraConfig } from "../../types/camera";
-import { ProximityState } from './proximityDetector';
+import type { CameraConfig, FaceRecognitionConfig } from "../../types/camera";
+export type ProximityState = 'near' | 'far' | 'stable';
 export declare class CameraSDK {
     private manager;
     private features;
     private container;
     private video;
     constructor(config: CameraConfig);
-    open(): Promise<void>;
+    open(): Promise<boolean>;
+    isOpen(): boolean;
     close(): void;
     takePhoto(): Promise<string>;
     switchCamera(facingMode: "user" | "environment"): Promise<void>;
-    detectPose(callback: (stable: ProximityState) => void): Promise<void>;
+    startFaceRecognition(config: FaceRecognitionConfig, callback: Function): Promise<boolean>;
+    stopFaceRecognition(): Promise<boolean>;
+    startBodyDetect(callback: (stable: ProximityState) => void): Promise<boolean>;
+    stopBodyDetect(): Promise<boolean>;
     destroy(): void;
 }

package/dist/types/modules/camera/mediaStream.d.ts

@@ -4,9 +4,10 @@ export declare class MediaStreamManager implements CameraCore {
     private stream;
     private facingMode;
     constructor(video: HTMLVideoElement, facingMode?: FacingMode);
-    open(): Promise<void>;
+    open(): Promise<boolean>;
     close(): void;
-    switch(facingMode: FacingMode): Promise<void>;
+    switch(facingMode: FacingMode): Promise<boolean>;
     getVideoEl(): HTMLVideoElement;
     getStream(): MediaStream | null;
+    isOpen(): boolean;
 }

package/dist/types/modules/core/voice-interaction.d.ts

@@ -37,7 +37,7 @@ declare class VoiceInteraction extends Emittery {
     private wakeWordDetection;
     private exitWordDetection;
     private interruptWordDetection;
-    onWakeDetected(): void;
+    onWakeDetected(echo?: any): void;
     onSleepDetected(): void;
     private onInterruptDetected;
     destroy(): void;

package/dist/types/modules/human.d.ts

@@ -6,7 +6,7 @@ import type { QuanTe } from '@/types/index';
 import { IResultData } from '@/api/interface';
 import voiceInteraction from './core/voice-interaction';
 import FramePlayer from '@/utils/framePlayer';
-import { CameraConfig } from '@/types/camera';
+import { CameraConfig, FaceRecognitionConfig } from '@/types/camera';
 import { CameraSDK } from './camera';
 declare class Human extends Emittery implements QuanTe.Human {
     name: string;
@@ -78,7 +78,11 @@ declare class Human extends Emittery implements QuanTe.Human {
     closeRecorder(): Promise<boolean>;
     openRecorder(): Promise<QuanTe.Result<QuanTe.Recorder.PermissionResult>>;
     openCamera(config: CameraConfig): Promise<boolean>;
-    wakeUp(isWake?: boolean): Promise<void>;
+    wakeUp(isWake: boolean | undefined, echo: any): Promise<void>;
+    startFaceRecognition(config?: FaceRecognitionConfig): Promise<boolean>;
+    stopFaceRecognition(): Promise<boolean>;
+    startBodyDetect(): Promise<boolean>;
+    stopBodyDetect(): Promise<boolean>;
 }
 export default Human;
 export type { QuanTe };

package/dist/types/types/camera.d.ts

@@ -16,4 +16,17 @@ export interface CameraFeature {
 export interface CameraCore {
   getVideoEl(): HTMLVideoElement;
   getStream(): MediaStream | null;
+}
+
+export interface FaceRecognitionConfig {
+  // 多长时间识别一次 单位 ms, 默认 3000ms
+  duration?: number;
+  // 识别人数 默认 不限
+  maxResults?: number;
+  // 是否在视频上绘制人脸检测框，默认 false
+  drawDetections?: boolean;
+  // 是否在视频上绘制人脸特征点，默认 false
+  drawFaceLandmarks?: boolean;
+  // 最小置信度，默认 0.6
+  minConfidence?: number; 
 }

package/dist/types/types/enum.d.ts

@@ -17,5 +17,6 @@ export declare enum EmitEvent {
     WAKE_RECORDER_STOP = "wake.record.stop",
     AUDIO_PAUSE = "audio.pause",
     AUDIO_PLAY = "audio.play",
-    CAMERA_PERSON_TEST_RESULT = "camera.person.test.result"
+    CAMERA_PERSON_TEST_RESULT = "camera.person.test.result",
+    CAMERA_FACE_RECOGNITION_RESULT = "camera.face.recognition.result"
 }

package/dist/types/types/enum.ts

@@ -36,5 +36,7 @@ export enum EmitEvent {
   // 播放音频
   AUDIO_PLAY = 'audio.play',
   // 摄像机人体检测结果
-  CAMERA_PERSON_TEST_RESULT = 'camera.person.test.result'
+  CAMERA_PERSON_TEST_RESULT = 'camera.person.test.result',
+  // 摄像机人脸识别结果
+  CAMERA_FACE_RECOGNITION_RESULT = 'camera.face.recognition.result'
 }

package/dist/types/types/index.d.ts

@@ -1,7 +1,7 @@
 import { API } from "./api";
 import * as THREE from 'three'
 import { GLTF } from 'three/examples/jsm/loaders/GLTFLoader.js';
-import { CameraConfig } from "./camera";
+import { CameraConfig, FaceRecognitionConfig } from "./camera";
 
 export namespace QuanTe {
   export type Character = 'xiaomeng' | 'custom' | 'quan' | 'doctor' | 'boy' | 'girl' | '';
@@ -113,6 +113,10 @@ export namespace QuanTe {
     public openRecorder(): Promise<QuanTe.Result<QuanTe.Recorder.PermissionResult>>
     // 打开摄像头
     public openCamera(config: CameraConfig): Promise<boolean>
+    public startFaceRecognition(config: FaceRecognitionConfig): Promise<boolean>
+    public stopFaceRecognition(): Promise<boolean>
+    public startBodyDetect(): Promise<boolean>
+    public stopBodyDetect(): Promise<boolean>
   }
 
   export interface ModelInfo {

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "qt-human",
-  "version": "2.3.5-alpha",
+  "version": "2.3.6-alpha",
   "main": "dist/bundle.umd.js",
   "module": "dist/bundle.esm.js",
   "types": "dist/types/index.d.ts",
@@ -59,11 +59,11 @@
   "dependencies": {
     "@mediapipe/pose": "^0.5.1675469404",
     "@tensorflow-models/pose-detection": "^2.1.3",
-    "@tensorflow/tfjs-backend-cpu": "^4.22.0",
     "@tensorflow/tfjs-backend-webgl": "^4.22.0",
     "@tensorflow/tfjs-backend-webgpu": "^4.22.0",
     "@tensorflow/tfjs-converter": "^4.22.0",
     "@tensorflow/tfjs-core": "^4.22.0",
+    "@vladmandic/face-api": "^1.7.15",
     "axios": "^1.7.2",
     "crypto-js": "^4.2.0",
     "emittery": "^1.0.3",