qt-human 2.2.0-alpha.5 → 2.2.0-alpha.6

package/dist/types/api/helper/index.d.ts

@@ -0,0 +1,2 @@
+export declare const CODE_SUCCESS: number;
+export declare const CODE_TOKEN_FAIL: number;

package/dist/types/api/human.d.ts

@@ -0,0 +1,20 @@
+import { SSE } from 'sse.js';
+import { QuanTe } from '@/types';
+import { IResultData } from './interface';
+import { CancelTokenSource } from 'axios';
+export declare const abortAll: () => void;
+export declare const fetchFloat32Array: (url?: string, params?: {}, isMulti?: boolean) => Promise<Float32Array>;
+export type Text2SpeechParams = {
+    role?: string;
+    speed?: number;
+    sn?: number;
+    model?: 'paddle2' | 'paddle' | 'azure';
+    viseme?: 'a2f' | 'azure' | 'none' | 'qt' | 'ovr';
+    smooth_method?: number;
+    smooth_interations?: number;
+};
+export declare const getCharacterInfoApi: (id: string) => Promise<IResultData<QuanTe.API.CharacterInfo>>;
+export declare const apiAsk: (characterId: string, params: QuanTe.API.ChatMessageParams, callback?: (res: IResultData<QuanTe.API.ChatMessage>) => void) => Promise<void>;
+export declare const apiText2Speech: (characterId: string, params: QuanTe.API.Text2SpeechParams) => Promise<IResultData<QuanTe.API.Text2Speech>>;
+export declare const getFetchSource: () => (AbortController | CancelTokenSource | SSE)[];
+export declare const getGatewayTokenApi: () => Promise<IResultData<string>>;

package/dist/types/api/interface/index.d.ts

@@ -0,0 +1,35 @@
+export interface IResult {
+    code: number;
+    msg: string;
+}
+export interface IResultData<T = any> extends IResult {
+    data: T;
+}
+export interface IPage<T = any> {
+    current: number;
+    limit: number;
+    totalPage: number;
+    total: number;
+    rows: T[];
+    param?: {
+        [key: string]: any;
+    } | string;
+}
+export interface IPageQuery {
+    page: number;
+    limit: number;
+}
+export interface IResultAny {
+    code: number;
+    msg: string;
+    [key: string]: any;
+}
+export type SystemBoolean = 'T' | 'F';
+export type Status = 'enable' | 'disable';
+export type ChatPermission = 'all_team_members' | 'only_me' | 'partial_members';
+export type ChatIndexingTechnique = 'high_quality' | 'economy';
+export type ChatDataSourceType = 'upload_file' | null;
+export type ChatIndexingStatus = 'waiting' | 'complete' | 'indexing';
+export type ChatDisplayStatus = 'disabled' | 'available' | 'indexing' | 'error';
+export type ClientStatus = 'enable' | 'unenable';
+export type GrantType = 'password' | 'sms' | 'email' | 'serectkey';

package/dist/types/const.d.ts

@@ -0,0 +1,12 @@
+export declare const AUTH_TOKEN_KEY = "auth-token";
+export declare const WSS_URL = "wss-url";
+export declare const PLATFORM_BASE_API = "platform-base-api";
+export declare const GATEWAY_TOKEN = "apigateway-token";
+export declare const APPID = "appid";
+export declare const BLANDSHAPES = "blocksShapes";
+export declare const AUDIO_URLS = "audioUrls";
+export declare const SEARCH_TEXT = "searchText";
+export declare const ASK_RESPONSE = "askResponse";
+export declare const API_GATEWAY_BASE_URL = "API_GATEWAY_BASE_URL";
+export declare const API_GPT_BASE_URL = "API_GPT_BASE_URL";
+export declare const API_FACEGOOD_BASE_URL = "API_FACEGOOD_BASE_URL";

package/dist/types/index.d.ts

@@ -0,0 +1,6 @@
+import Human from './modules/human';
+import { QuanTe } from './types';
+import { EmitEvent } from './types/enum';
+export type { QuanTe };
+export { EmitEvent };
+export default Human;

package/dist/types/modules/components/loading.d.ts

@@ -0,0 +1,37 @@
+import * as THREE from 'three';
+import Human from '../human';
+import { QuanTe } from '@/types';
+declare class Loading {
+    hm: Human;
+    animationFrameId: number;
+    container: HTMLElement;
+    scene: THREE.Scene;
+    width: number;
+    height: number;
+    depth: number;
+    zAxisNumber: number;
+    camera: THREE.PerspectiveCamera;
+    renderer: THREE.WebGLRenderer;
+    sphere: any;
+    parameters: any[];
+    materials: any[];
+    particles_init_postion: number;
+    zprogress: number;
+    zprogress_second: number;
+    particles_first: any[];
+    particles_second: any[];
+    constructor(container: string | HTMLElement, hm: Human);
+    initProgressBar(): void;
+    initScene(): void;
+    initCamera(): void;
+    initRenderer(): void;
+    initOrbitControls(): void;
+    animate(): void;
+    initLight(): void;
+    renderSphereRotate(): void;
+    initSceneStar(initZpostion: number): THREE.Points<THREE.BufferGeometry<THREE.NormalBufferAttributes>, any, THREE.Object3DEventMap>[];
+    renderStarMove(): void;
+    onProgress({ code, progress, total }: QuanTe.Loader.Progress): void;
+    destroy(): void;
+}
+export default Loading;

package/dist/types/modules/core/audio.d.ts

@@ -0,0 +1,27 @@
+import Emittery from 'emittery';
+declare class QtAudio extends Emittery {
+    private url;
+    private audio;
+    static VOLUMN: number;
+    static MUTED: boolean;
+    constructor(url?: string);
+    setUrl(url: string): Promise<boolean>;
+    getUrl(): string;
+    play(): Promise<boolean>;
+    restart(): Promise<boolean>;
+    pause(): void;
+    seek(time: number): void;
+    load(): void;
+    reset(): void;
+    isLoaded(): boolean;
+    isPlaying(): boolean;
+    getDuration(): number;
+    setVolume(volume: number): void;
+    getVolume(): number;
+    destroy(): void;
+    mute(isMute?: boolean): void;
+}
+declare const MutilAudio: typeof QtAudio;
+declare const _default: QtAudio;
+export default _default;
+export { type QtAudio, MutilAudio };

package/dist/types/modules/core/loader.d.ts

@@ -0,0 +1,8 @@
+import 'blob-polyfill';
+import type { QuanTe } from "@/types";
+declare const loadBundleFile: (params: {
+    model: QuanTe.API.CharacterModel;
+    onProgress: (url: string, progress: number, total: number) => void;
+}) => Promise<QuanTe.Loader.LoaderResponse>;
+declare const loadModel: (models: QuanTe.API.CharacterModel[], onProgress?: (d: number) => void) => Promise<QuanTe.Loader.LoaderResponse[]>;
+export { loadModel, loadBundleFile };

package/dist/types/modules/core/recorder.d.ts

@@ -0,0 +1,26 @@
+import Recorder from 'recorder-core';
+import 'recorder-core/src/engine/pcm.js';
+import QtWebSocket from './websocket';
+import Emittery from 'emittery';
+import type { QuanTe } from '@/types/index';
+declare class QtRecorder extends Emittery {
+    recorder: typeof Recorder;
+    socket: QtWebSocket | null;
+    private audioBuffer;
+    Recorder: any;
+    private mediaStream;
+    private firstRequestPermision;
+    constructor();
+    start(token?: string): Promise<QuanTe.Result<QuanTe.Recorder.PermissionResult>>;
+    stop(): Promise<QuanTe.Result<null>>;
+    closeRecording(): Promise<any>;
+    requestRecorderPermission(): Promise<QuanTe.Result<QuanTe.Recorder.PermissionResult>>;
+    isOpen(): any;
+    resume(): void;
+    private connectSocket;
+    private onProcess;
+    open(): Promise<QuanTe.Result<QuanTe.Recorder.PermissionResult>>;
+    close(): Promise<QuanTe.Result<null>>;
+    destroy(): void;
+}
+export default QtRecorder;

package/dist/types/modules/core/recorder2.d.ts

@@ -0,0 +1,38 @@
+import Recorder from 'recorder-core';
+import 'recorder-core/src/engine/pcm.js';
+import QtWebSocket from './websocket';
+import Emittery from 'emittery';
+import type { QuanTe } from '@/types/index';
+declare class QtRecorder extends Emittery {
+    recorder: typeof Recorder;
+    socket: QtWebSocket | null;
+    private audioBuffer;
+    Recorder: any;
+    private firstRequestPermision;
+    private isSending;
+    private timeout;
+    private config;
+    private readonly BUFFER_SIZE;
+    private receiveTimeout;
+    private waitTime;
+    constructor(config: {
+        token: string;
+        level: number;
+        waitTime?: number;
+    });
+    start(): void;
+    stop(): void;
+    closeRecording(): Promise<boolean>;
+    requestRecorderPermission(): Promise<QuanTe.Result<QuanTe.Recorder.PermissionResult>>;
+    isOpen(): any;
+    resume(): void;
+    private connectSocket;
+    private onProcess;
+    computeAudioEnergy(audioData: Int16Array): number;
+    addAudioBuffer(buffer: Int16Array): void;
+    dealAudioData(level: number, threshold?: number): void;
+    checkAudioActivityByRMS(int16Array: Int16Array, threshold?: number): boolean;
+    openRecording(): Promise<QuanTe.Result<QuanTe.Recorder.PermissionResult>>;
+    sendMessage(buffer: Int16Array): void;
+}
+export default QtRecorder;

package/dist/types/modules/core/render.d.ts

@@ -0,0 +1,117 @@
+import * as THREE from 'three';
+import { OrbitControls } from 'three/examples/jsm/controls/OrbitControls.js';
+import { GLTFLoader, GLTF } from 'three/examples/jsm/loaders/GLTFLoader.js';
+import { KTX2Loader } from 'three/examples/jsm/loaders/KTX2Loader.js';
+import Emittery from 'emittery';
+import type { QuanTe } from '@/types/index';
+type ActionObjType = {
+    action: THREE.AnimationAction;
+    name: string;
+    code: string;
+    animation: THREE.AnimationClip;
+    type: string;
+    text: string;
+};
+type BodyObject = {
+    [key: string]: QuanTe.Model.BodyOption & {
+        mesh: THREE.Mesh | undefined;
+        morphTargetInfluences?: number[] | null | undefined;
+    };
+};
+declare class ThreeJSApp extends Emittery {
+    animationFrameId: number;
+    container: HTMLElement | null;
+    scene: THREE.Scene | null;
+    camera: THREE.PerspectiveCamera | null;
+    renderer: THREE.WebGLRenderer | null;
+    loader: GLTFLoader | null;
+    ktx2Loader: KTX2Loader | null;
+    controls: OrbitControls | null;
+    width: number;
+    height: number;
+    depth: number;
+    config: QuanTe.Model.Description;
+    model: {
+        [key: string]: GLTF;
+    };
+    mixer: THREE.AnimationMixer | null;
+    animations: {
+        code: string;
+        type: string;
+        animation: THREE.AnimationClip;
+        text: string;
+    }[];
+    actions: ActionObjType[];
+    isNotRender: boolean;
+    animationMixer: THREE.AnimationMixer | null;
+    clock: THREE.Clock | null;
+    body: BodyObject;
+    characterGroup: THREE.Group | null;
+    lightGroup: THREE.Group | null;
+    state: {
+        [key: string]: any;
+    };
+    private onWindowResizeBound;
+    private onAnimationFinishedBound;
+    private actionDefalutMap;
+    animationGroup: THREE.AnimationObjectGroup;
+    rotationCenter: THREE.Vector3;
+    private muteActionState;
+    private actionNotifier;
+    private _container;
+    constructor(container: (string | HTMLElement) | undefined, config: QuanTe.Model.Description);
+    init(): void;
+    setContainer(): void;
+    updateLight(): void;
+    initLight(): void;
+    degree2Euler(xyz: QuanTe.Model.XYZ): QuanTe.Model.XYZ;
+    private dataToMatrix4;
+    setLight(light: QuanTe.Model.Light): void;
+    setAngleView({ camera }: {
+        camera: QuanTe.Model.TCamera & {
+            duration?: number;
+        };
+    }): void;
+    gsapContainer(opts: {
+        attr?: {
+            [key: string]: any;
+        };
+    }): Promise<unknown>;
+    initOrbitControls(): void;
+    setBackground(bg?: string): void;
+    loadGLBModel(glbURL: string): void;
+    loadLowResTexture(url: string): Promise<unknown>;
+    private cloneTrack;
+    changeBundle(record: QuanTe.Loader.LoaderResponse): void;
+    loadBundles(list: QuanTe.Loader.LoaderResponse[], onProgress: (progress: number) => void): Promise<boolean>;
+    private animationFinished;
+    onWindowResize: import("lodash").DebouncedFuncLeading<() => void>;
+    setCamera(options: QuanTe.Model.TCamera): void;
+    animate(): void;
+    startRender(): void;
+    stopRender(): void;
+    gsapMorph(target: number[], source: number[], t2: any): void;
+    initIdle: import("lodash").DebouncedFunc<() => void>;
+    playAction: import("lodash").DebouncedFuncLeading<(code: string, opts?: QuanTe.PlayActionOptions) => void>;
+    stopAllActionExcludeIdle(): void;
+    stopAllAction(): void;
+    private updateControls;
+    private mousedown;
+    private mouseup;
+    private mousemove;
+    private registerListeners;
+    cleanMaterial(material: any): void;
+    destroy(): void;
+    playByShapes(shapes?: number[], idx?: number): Promise<void>;
+    addModel(code: string, modelUrl: string): void;
+    destoryModel(code: string): void;
+    muteAction(isMute?: boolean): void;
+    getModelInfo(): QuanTe.ModelInfo;
+    loadGlb(url: string, onProgress?: (event: ProgressEvent) => void): Promise<GLTF>;
+    addGlb(url: string, onProgress?: (event: ProgressEvent) => void): Promise<GLTF>;
+    setCharacterCenter(): void;
+    setVideoBackground(videoURL: string): void;
+    loadCubeBackground(imageURLs: string[]): void;
+    setBackgroundColorOrGradient(colorOrGradient: string): void;
+}
+export default ThreeJSApp;

package/dist/types/modules/core/voice-interaction.d.ts

@@ -0,0 +1,46 @@
+import Emittery from 'emittery';
+import QtRecorder from './recorder2';
+export declare enum VoiceState {
+    SLEEP = "SLEEP",
+    LISTENING = "LISTENING",
+    RECORDING = "RECORDING",
+    PROCESSING = "PROCESSING",
+    SPEAKING = "SPEAKING"
+}
+declare class VoiceInteraction extends Emittery {
+    private isInitialized;
+    recorder?: QtRecorder;
+    private wakeFuse?;
+    private exitFuse?;
+    private interruptFuse?;
+    private config;
+    private currentState;
+    private isWake;
+    private lastRecordTime;
+    private EXIT_TIME;
+    private taskInterval?;
+    private onAnyEvent;
+    private keepActive;
+    constructor();
+    initialize(config: {
+        asrEndpoint: string;
+        wakeWords: string[];
+        exitWords: string[];
+        interruptWorlds: string[];
+        sampleRate: number;
+        gatewayToken: string;
+        level: number;
+        exitTime: number;
+        keepActive: boolean;
+        waitTime?: number;
+    }): Promise<void>;
+    private wakeWordDetection;
+    private exitWordDetection;
+    private interruptWordDetection;
+    private onWakeDetected;
+    private onSleepDetected;
+    private onInterruptDetected;
+    destroy(): void;
+}
+declare const _default: VoiceInteraction;
+export default _default;

package/dist/types/modules/core/websocket.d.ts

@@ -0,0 +1,19 @@
+declare class QtWebSocket {
+    private url;
+    private ws;
+    private callback;
+    private isDestroyed;
+    constructor(url: string);
+    connect(config?: {
+        onopen?: () => void;
+        onerror?: (e: any) => void;
+        onclose?: (e: any) => void;
+        onmessage?: () => void;
+    }): Promise<void>;
+    send(data: any): Promise<void>;
+    close(): void;
+    onMessage(callback: (data: any) => void): void;
+    isReady(): boolean;
+    destroy(): void;
+}
+export default QtWebSocket;

package/dist/types/modules/human.d.ts

@@ -0,0 +1,78 @@
+import Emittery from 'emittery';
+import ThreeJSApp from './core/render';
+import Loading from './components/loading';
+import Recorder from './core/recorder';
+import type { QuanTe } from '@/types/index';
+import { IResultData } from '@/api/interface';
+import voiceInteraction from './core/voice-interaction';
+declare class Human extends Emittery implements QuanTe.Human {
+    name: string;
+    core: ThreeJSApp | null;
+    private config;
+    loading?: Loading;
+    private audio;
+    private container;
+    recorder: Recorder | null;
+    private voiceCallback;
+    private status;
+    private probeTimeout;
+    private callbackTimeout;
+    private mutilAudios;
+    private exeSequence;
+    private fetchSequence;
+    private openRecorderTimeout;
+    private onAnyEvent;
+    private characterData;
+    private isNotRender;
+    private mode;
+    private ttsclient;
+    private framePlayer;
+    voiceInter: typeof voiceInteraction;
+    constructor(config: QuanTe.Configuration);
+    private refreshGatewayToken;
+    private initTTSClient;
+    setCharacterData(characterId: string): Promise<boolean>;
+    setCharacter(characterId: string, configuration?: Partial<QuanTe.Configuration>): Promise<boolean>;
+    speak(text: string, callback?: Function, opt?: {
+        [key: string]: any;
+    }): Promise<QuanTe.API.StreamAudioPlayInfo>;
+    private _speak;
+    askCommand(params: QuanTe.API.AskParams, callback: (msgs: IResultData<string[]>) => void): Promise<void>;
+    ask(params: QuanTe.API.AskParams, callback: (msgs: IResultData<QuanTe.API.ChatMessage[]>) => void): Promise<QuanTe.Result<QuanTe.API.AskResponse>>;
+    private _textToSpeech;
+    askStream(params: QuanTe.API.AskParams, callback: (msgs: IResultData<QuanTe.API.ChatMessage[]>) => void): Promise<QuanTe.Result<QuanTe.API.AskStreamResponse>>;
+    private initBlendShapes;
+    private probeTimePlay;
+    speakByAudioShapes(uris: string[], shapes: Array<number[][]>): Promise<boolean>;
+    speakByAudioStreamShapes(audios: ArrayBuffer[], shapes: number[][], rate?: number): Promise<boolean>;
+    private createAudio;
+    speakByAudio(uri: string, shapes: number[][]): Promise<boolean>;
+    private setRecorderStatus;
+    private play;
+    startVoice2Text(callback?: (data: QuanTe.Recorder.Voice2Text) => void): Promise<QuanTe.API.IResultData<QuanTe.Recorder.PermissionResult>>;
+    stopVoice2Text(callback?: (data: QuanTe.Recorder.Voice2Text) => void): Promise<QuanTe.Result<null>>;
+    stop(): Promise<boolean>;
+    getStatus(): QuanTe.Status;
+    private initStorage;
+    muteAudio(isMute?: boolean): void;
+    muteAction(isMute?: boolean): void;
+    private dealVoice2Text;
+    destroy(): Promise<void>;
+    updateToken(token: string): void;
+    setAngleView({ camera }: {
+        camera: QuanTe.Model.TCamera & {
+            duration?: number;
+        };
+    }): void;
+    playAction(code: string, opts?: QuanTe.PlayActionOptions): Promise<void>;
+    getModelInfo(): QuanTe.ModelInfo | null;
+    openVoiceInteraction(config: QuanTe.VoiceConfig): Promise<boolean>;
+    private voiceInteractionEmit;
+    offlineRender(files: QuanTe.API.CharacterModel[]): Promise<boolean>;
+    startRender(): Promise<boolean>;
+    stopRender(): Promise<boolean>;
+    closeRecorder(): Promise<boolean>;
+    openRecorder(): Promise<QuanTe.Result<QuanTe.Recorder.PermissionResult>>;
+}
+export default Human;
+export type { QuanTe };

package/dist/types/types/declarations.d.ts

@@ -0,0 +1,4 @@
+declare module '*.wav' {
+  const value: string;
+  export default value;
+}

package/dist/types/types/enum.d.ts

@@ -0,0 +1,20 @@
+export declare enum EmitEvent {
+    HUMAN_READY = "human.ready",
+    HUMAN_ERROR = "human.error",
+    HUMAN_ACTION_PENDING = "human.action.pending",
+    HUMAN_LOAD_PROGRESS = "human.load.progress",
+    CORE_BUNDLES_LOADED = "core.bundles.loaded",
+    RECORDER_MESSAGE = "recorder.message",
+    RECORDER_START = "recorder.start",
+    RECORDER_STOP = "recorder.stop",
+    RECORDER_SENDING = "recorder.sending",
+    VOICE_SLEEP = "voice.sleep",
+    VOICE_WAKE = "voice.wake",
+    VOICE_INTERRUPT = "voice.interrupt",
+    VOICE2TEXT = "voice2text",
+    ASRTEXT = "asrtext",
+    WAKE_RECORDER_START = "wake.record.start",
+    WAKE_RECORDER_STOP = "wake.record.stop",
+    AUDIO_PAUSE = "audio.pause",
+    AUDIO_PLAY = "audio.play"
+}

package/dist/types/types/enum.ts

@@ -0,0 +1,38 @@
+export enum EmitEvent {
+  // human对象加载完成
+  HUMAN_READY = 'human.ready',
+  // 错误
+  HUMAN_ERROR = 'human.error',
+  // 触发指令
+  HUMAN_ACTION_PENDING = 'human.action.pending',
+  // 加载进度
+  HUMAN_LOAD_PROGRESS = 'human.load.progress',
+  // 模型加载完成
+  CORE_BUNDLES_LOADED = 'core.bundles.loaded',
+  // 录音消息
+  RECORDER_MESSAGE = 'recorder.message',
+  // 录音开始
+  RECORDER_START = 'recorder.start',
+  // 录音结束
+  RECORDER_STOP = 'recorder.stop',
+  // 发送录音数据（ASR）
+  RECORDER_SENDING = 'recorder.sending',
+  // 语音-睡眠模式
+  VOICE_SLEEP = 'voice.sleep',
+  // 语音-唤醒模式
+  VOICE_WAKE = 'voice.wake',
+  // 语音-打断
+  VOICE_INTERRUPT = 'voice.interrupt',
+  // 语音转文本
+  VOICE2TEXT = 'voice2text',
+  // asr文本数据
+  ASRTEXT = 'asrtext',
+  // 唤醒状态下开始录音
+  WAKE_RECORDER_START = 'wake.record.start',
+  // 唤醒状态下结束录音
+  WAKE_RECORDER_STOP = 'wake.record.stop',
+  // 音频暂停播放
+  AUDIO_PAUSE = 'audio.pause',
+  // 播放音频
+  AUDIO_PLAY = 'audio.play',
+}