speechstate 2.0.0-beta.0

package/dist/speechstate.js

@@ -0,0 +1,235 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.speechstate = void 0;
+const xstate_1 = require("xstate");
+const tts_1 = require("./tts");
+const asr_1 = require("./asr");
+const speechstate = (0, xstate_1.createMachine)({
+    types: {},
+    context: ({ input }) => ({
+        settings: input,
+    }),
+    id: "speechstate",
+    type: "parallel",
+    states: {
+        asrTtsSpawner: {
+            initial: "idle",
+            states: {
+                idle: { on: { PREPARE: "createAudioContext" } },
+                createAudioContext: {
+                    invoke: {
+                        id: "createAudioContext",
+                        src: "audioContext",
+                        onDone: {
+                            target: "spawn",
+                            actions: (0, xstate_1.assign)({ audioContext: ({ event }) => event.output }),
+                        },
+                    },
+                },
+                spawn: {
+                    entry: [
+                        (0, xstate_1.assign)({
+                            ttsRef: ({ context, spawn }) => {
+                                return spawn(tts_1.ttsMachine, {
+                                    input: {
+                                        ttsDefaultVoice: context.settings.ttsDefaultVoice,
+                                        audioContext: context.audioContext,
+                                        azureCredentials: context.settings.azureCredentials,
+                                    },
+                                });
+                            },
+                        }),
+                        (0, xstate_1.assign)({
+                            asrRef: ({ context, spawn }) => {
+                                return spawn(asr_1.asrMachine, {
+                                    input: {
+                                        asrDefaultCompleteTimeout: context.settings.asrDefaultCompleteTimeout,
+                                        asrDefaultNoInputTimeout: context.settings.asrDefaultNoInputTimeout,
+                                        locale: context.settings.locale,
+                                        audioContext: context.audioContext,
+                                        azureCredentials: context.settings.azureCredentials,
+                                    },
+                                });
+                            },
+                        }),
+                    ],
+                    after: {
+                        300000: {
+                            target: "spawn",
+                        },
+                    },
+                },
+            },
+        },
+        asrTtsManager: {
+            initial: "initialize",
+            on: {
+                TTS_READY: {
+                    actions: () => console.debug("[TTS→SpSt] TTS_READY"),
+                    target: ".preReady",
+                },
+                ASR_READY: {
+                    actions: () => console.debug("[ASR→SpSt] ASR_READY"),
+                    target: ".preReady",
+                },
+                // ASR_ERROR not implemented
+                TTS_ERROR: {
+                    actions: () => console.error("[TTS→SpSt] TTS_ERROR"),
+                    target: ".fail",
+                },
+                ASR_NOINPUT: {
+                    actions: [
+                        () => console.debug("[ASR→SpSt] NOINPUT"),
+                        (0, xstate_1.sendParent)({ type: "ASR_NOINPUT" }),
+                    ],
+                    target: ".ready",
+                },
+            },
+            states: {
+                initialize: {},
+                preReady: {
+                    on: {
+                        TTS_READY: {
+                            actions: () => console.debug("[TTS→SpSt] TTS_READY"),
+                            target: "ready",
+                        },
+                        ASR_READY: {
+                            actions: () => console.debug("[ASR→SpSt] ASR_READY"),
+                            target: "ready",
+                        },
+                    },
+                },
+                ready: {
+                    initial: "idle",
+                    entry: [
+                        () => console.debug("[SpSt] All ready"),
+                        (0, xstate_1.sendParent)({ type: "ASRTTS_READY" }),
+                    ],
+                    states: {
+                        idle: {
+                            on: {
+                                LISTEN: { target: "waitForRecogniser" },
+                                SPEAK: [
+                                    {
+                                        target: "speaking",
+                                    },
+                                ],
+                            },
+                        },
+                        speaking: {
+                            entry: [
+                                ({ event }) => console.debug("[SpSt→TTS] SPEAK", event.value),
+                                ({ context, event }) => context.ttsRef.send({
+                                    type: "SPEAK",
+                                    value: event.value,
+                                }),
+                            ],
+                            on: {
+                                CONTROL: {
+                                    actions: [
+                                        () => console.debug("[SpSt→TTS] CONTROL"),
+                                        ({ context }) => context.ttsRef.send({
+                                            type: "CONTROL",
+                                        }),
+                                    ],
+                                },
+                                STOP: {
+                                    actions: [
+                                        () => console.debug("[SpSt→TTS] STOP"),
+                                        ({ context }) => context.ttsRef.send({
+                                            type: "STOP",
+                                        }),
+                                    ],
+                                },
+                                TTS_STARTED: {
+                                    actions: [
+                                        () => console.debug("[TTS→SpSt] TTS_STARTED"),
+                                        (0, xstate_1.sendParent)({ type: "TTS_STARTED" }),
+                                    ],
+                                },
+                                SPEAK_COMPLETE: {
+                                    target: "idle",
+                                    actions: [
+                                        () => console.debug("[TTS→SpSt] SPEAK_COMPLETE"),
+                                        (0, xstate_1.sendParent)({ type: "SPEAK_COMPLETE" }),
+                                    ],
+                                },
+                            },
+                        },
+                        waitForRecogniser: {
+                            entry: [
+                                ({ event }) => console.debug("[SpSt→ASR] START", event.value),
+                                ({ context, event }) => context.asrRef.send({
+                                    type: "START",
+                                    value: event.value,
+                                }),
+                            ],
+                            on: {
+                                ASR_STARTED: {
+                                    target: "recognising",
+                                    actions: [
+                                        () => console.debug("[ASR→SpSt] ASR_STARTED"),
+                                        (0, xstate_1.sendParent)({ type: "ASR_STARTED" }),
+                                    ],
+                                },
+                            },
+                        },
+                        recognising: {
+                            on: {
+                                CONTROL: {
+                                    actions: [
+                                        () => console.debug("[SpSt→ASR] CONTROL"),
+                                        ({ context }) => context.asrRef.send({
+                                            type: "CONTROL",
+                                        }),
+                                    ],
+                                },
+                                RECOGNISED: {
+                                    actions: [
+                                        ({ event }) => console.debug("[ASR→SpSt] RECOGNISED", event.value),
+                                        (0, xstate_1.sendParent)(({ event }) => ({
+                                            type: "RECOGNISED",
+                                            value: event.value,
+                                        })),
+                                    ],
+                                    target: "idle",
+                                },
+                            },
+                        },
+                    },
+                },
+                fail: {},
+            },
+        },
+    },
+}, {
+    actors: {
+        audioContext: (0, xstate_1.fromPromise)(() => {
+            const audioContext = new (window.AudioContext ||
+                window.webkitAudioContext)();
+            navigator.mediaDevices
+                .getUserMedia({ audio: true })
+                .then(function (stream) {
+                audioContext.createMediaStreamSource(stream);
+            });
+            return audioContext;
+        }),
+    },
+    actions: {
+    //   logRecResult: ({ event }) => {
+    //     console.log("U>", (event as any).value[0]["utterance"], {
+    //       confidence: (event as any).value[0]["confidence"],
+    //     });
+    //   },
+    //   logAgenda: ({ context, event }) => {
+    //     console.log("S>", (event as any).value, {
+    //       passivity: `${context.tdmPassivity ?? "∞"} ms`,
+    //       speechCompleteTimeout: `${
+    //         context.tdmSpeechCompleteTimeout ||
+    //         context.settings.completeTimeout
+    //       } ms`,
+    //     });
+    //   },
+    },
+});
+exports.speechstate = speechstate;

package/dist/tts.d.ts

@@ -0,0 +1,42 @@
+import { AzureCredentials, Agenda } from "./types";
+interface MySpeechSynthesisUtterance extends SpeechSynthesisUtterance {
+    new (s: string): any;
+}
+interface TTSInit {
+    audioContext: AudioContext;
+    azureCredentials: string | AzureCredentials;
+    ttsDefaultVoice: string;
+}
+interface TTSContext extends TTSInit {
+    azureAuthorizationToken?: string;
+    ttsLexicon?: string;
+    wsaTTS?: SpeechSynthesis;
+    wsaVoice?: SpeechSynthesisVoice;
+    wsaUtt?: MySpeechSynthesisUtterance;
+    agenda?: Agenda;
+}
+type TTSEvent = {
+    type: "PREPARE";
+} | {
+    type: "CONTROL";
+} | {
+    type: "STOP";
+} | {
+    type: "READY";
+    value: {
+        wsaTTS: SpeechSynthesis;
+        wsaUtt: MySpeechSynthesisUtterance;
+    };
+} | {
+    type: "ERROR";
+} | {
+    type: "SPEAK";
+    value: Agenda;
+} | {
+    type: "TTS_STARTED";
+} | {
+    type: "SPEAK_COMPLETE";
+};
+export declare const ttsMachine: import("xstate").StateMachine<TTSContext, TTSEvent, Record<string, import("xstate").AnyActorRef>, import("xstate").ProvidedActor, import("xstate").ParameterizedObject, import("xstate").ParameterizedObject, string, import("xstate").StateValue, string, TTSInit, {}, import("xstate").ResolveTypegenMeta<import("xstate").TypegenDisabled, TTSEvent, import("xstate").ProvidedActor, import("xstate").ParameterizedObject, import("xstate").ParameterizedObject, string, string>>;
+export {};
+//# sourceMappingURL=tts.d.ts.map

package/dist/tts.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"tts.d.ts","sourceRoot":"","sources":["../src/tts.ts"],"names":[],"mappings":"AAMA,OAAO,EAAE,gBAAgB,EAAE,MAAM,EAAE,MAAM,SAAS,CAAC;AAEnD,UAAU,0BAA2B,SAAQ,wBAAwB;IACnE,KAAK,CAAC,EAAE,MAAM,OAAE;CACjB;AAED,UAAU,OAAO;IACf,YAAY,EAAE,YAAY,CAAC;IAC3B,gBAAgB,EAAE,MAAM,GAAG,gBAAgB,CAAC;IAC5C,eAAe,EAAE,MAAM,CAAC;CACzB;AAED,UAAU,UAAW,SAAQ,OAAO;IAClC,uBAAuB,CAAC,EAAE,MAAM,CAAC;IACjC,UAAU,CAAC,EAAE,MAAM,CAAC;IACpB,MAAM,CAAC,EAAE,eAAe,CAAC;IACzB,QAAQ,CAAC,EAAE,oBAAoB,CAAC;IAChC,MAAM,CAAC,EAAE,0BAA0B,CAAC;IACpC,MAAM,CAAC,EAAE,MAAM,CAAC;CACjB;AAED,KAAK,QAAQ,GACT;IAAE,IAAI,EAAE,SAAS,CAAA;CAAE,GACnB;IAAE,IAAI,EAAE,SAAS,CAAA;CAAE,GACnB;IAAE,IAAI,EAAE,MAAM,CAAA;CAAE,GAChB;IACE,IAAI,EAAE,OAAO,CAAC;IACd,KAAK,EAAE;QACL,MAAM,EAAE,eAAe,CAAC;QACxB,MAAM,EAAE,0BAA0B,CAAC;KACpC,CAAC;CACH,GACD;IAAE,IAAI,EAAE,OAAO,CAAA;CAAE,GACjB;IAAE,IAAI,EAAE,OAAO,CAAC;IAAC,KAAK,EAAE,MAAM,CAAA;CAAE,GAChC;IAAE,IAAI,EAAE,aAAa,CAAA;CAAE,GACvB;IAAE,IAAI,EAAE,gBAAgB,CAAA;CAAE,CAAC;AAE/B,eAAO,MAAM,UAAU,sdA+KtB,CAAC"}

package/dist/tts.js

@@ -0,0 +1,182 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.ttsMachine = void 0;
+const xstate_1 = require("xstate");
+const getToken_1 = require("./getToken");
+const TextToSpeech_1 = require("web-speech-cognitive-services/lib/SpeechServices/TextToSpeech");
+const REGION = "northeurope";
+exports.ttsMachine = (0, xstate_1.createMachine)({
+    id: "tts",
+    types: {},
+    context: ({ input }) => ({
+        ttsDefaultVoice: input.ttsDefaultVoice || "en-US-DavisNeural",
+        audioContext: input.audioContext,
+        azureCredentials: input.azureCredentials,
+    }),
+    initial: "GetToken",
+    on: {
+        READY: {
+            target: ".Ready",
+            actions: [
+                (0, xstate_1.assign)({
+                    wsaTTS: ({ event }) => event.value.wsaTTS,
+                    wsaUtt: ({ event }) => event.value.wsaUtt,
+                }),
+                (0, xstate_1.sendParent)({ type: "TTS_READY" }),
+            ],
+        },
+        ERROR: { actions: (0, xstate_1.sendParent)({ type: "TTS_ERROR" }) },
+    },
+    states: {
+        Ready: {
+            initial: "Idle",
+            states: {
+                Idle: {
+                    on: {
+                        SPEAK: {
+                            target: "Speaking",
+                            actions: (0, xstate_1.assign)({ agenda: ({ event }) => event.value }),
+                        },
+                    },
+                },
+                Speaking: {
+                    initial: "Go",
+                    on: {
+                        STOP: {
+                            target: "Idle",
+                        },
+                        TTS_STARTED: {
+                            actions: (0, xstate_1.sendParent)({ type: "TTS_STARTED" }),
+                        },
+                        SPEAK_COMPLETE: {
+                            target: "Idle",
+                        },
+                    },
+                    exit: (0, xstate_1.sendParent)({ type: "SPEAK_COMPLETE" }),
+                    states: {
+                        Go: {
+                            invoke: {
+                                src: "start",
+                                input: ({ context }) => ({
+                                    wsaTTS: context.wsaTTS,
+                                    wsaUtt: context.wsaUtt,
+                                    ttsLexicon: context.ttsLexicon,
+                                    voice: context.agenda.voice || context.ttsDefaultVoice,
+                                    // streamURL: context.agenda.streamURL,
+                                    utterance: context.agenda.utterance,
+                                }),
+                            },
+                            on: {
+                                CONTROL: "Paused",
+                            },
+                            exit: "ttsStop",
+                        },
+                        Paused: {
+                            on: {
+                                CONTROL: "Go",
+                            },
+                        },
+                    },
+                },
+            },
+        },
+        Fail: {},
+        GetToken: {
+            invoke: {
+                id: "getAuthorizationToken",
+                input: ({ context }) => ({
+                    credentials: context.azureCredentials,
+                }),
+                src: "getToken",
+                onDone: {
+                    target: "Ponyfill",
+                    actions: [
+                        (0, xstate_1.assign)(({ event }) => {
+                            return { azureAuthorizationToken: event.output };
+                        }),
+                    ],
+                },
+                onError: {
+                    actions: ({ event }) => console.error("[TTS] getToken error", event),
+                    target: "Fail",
+                },
+            },
+        },
+        Ponyfill: {
+            invoke: {
+                id: "ponyTTS",
+                src: "ponyfill",
+                input: ({ context }) => ({
+                    audioContext: context.audioContext,
+                    azureAuthorizationToken: context.azureAuthorizationToken,
+                }),
+            },
+        },
+    },
+}, {
+    actions: {
+        ttsStop: ({ context }) => {
+            context.wsaTTS.cancel();
+        },
+    },
+    actors: {
+        getToken: getToken_1.getToken,
+        ponyfill: (0, xstate_1.fromCallback)(({ sendBack, input }) => {
+            const ponyfill = (0, TextToSpeech_1.default)({
+                audioContext: input.audioContext,
+                credentials: {
+                    region: REGION,
+                    authorizationToken: input.azureAuthorizationToken,
+                },
+            });
+            const { speechSynthesis, SpeechSynthesisUtterance } = ponyfill;
+            const tts = speechSynthesis;
+            const ttsUtterance = SpeechSynthesisUtterance;
+            tts.addEventListener("voiceschanged", () => {
+                const voices = tts.getVoices();
+                if (voices.length > 0) {
+                    console.debug("[TTS] READY");
+                    sendBack({
+                        type: "READY",
+                        value: { wsaTTS: tts, wsaUtt: ttsUtterance },
+                    });
+                }
+                else {
+                    console.error("[TTS] No voices available");
+                    sendBack({ type: "ERROR" });
+                }
+            });
+        }),
+        start: (0, xstate_1.fromCallback)(({ sendBack, input }) => {
+            if (["", " "].includes(input.utterance)) {
+                console.debug("[TTS] SPEAK: ", input.utterance);
+                input.wsaTTS.speak("");
+            }
+            else {
+                console.debug("[TTS] SPEAK: ", input.utterance);
+                const content = wrapSSML(input.utterance, input.voice, input.ttsLexicon, 1); // todo speech rate;
+                const utterance = new input.wsaUtt(content);
+                utterance.addEventListener("start", () => {
+                    sendBack({ type: "TTS_STARTED" });
+                    console.debug("[TTS] TTS_STARTED");
+                });
+                utterance.addEventListener("end", () => {
+                    sendBack({ type: "SPEAK_COMPLETE" });
+                    console.debug("[TTS] SPEAK_COMPLETE");
+                });
+                input.wsaTTS.speak(utterance);
+            }
+        }),
+    },
+});
+const wrapSSML = (text, voice, lexicon, speechRate) => {
+    let content = `<speak version="1.0" xmlns="http://www.w3.org/2001/10/synthesis" xmlns:mstts="http://www.w3.org/2001/mstts" xml:lang="en-US"><voice name="${voice}">`;
+    if (lexicon) {
+        content = content + `<lexicon uri="${lexicon}"/>`;
+    }
+    content =
+        content +
+            `<prosody rate="${speechRate}">` +
+            `${text}</prosody></voice></speak>`;
+    return content;
+};

package/dist/types.d.ts

@@ -0,0 +1,27 @@
+export interface AzureCredentials {
+    endpoint: string;
+    key: string;
+}
+export interface Settings {
+    locale: string;
+    azureCredentials: string | AzureCredentials;
+    asrDefaultCompleteTimeout: number;
+    asrDefaultNoInputTimeout: number;
+    ttsDefaultVoice: string;
+}
+export interface Agenda {
+    utterance: string;
+    voice?: string;
+    stream?: ReadableStream<string>;
+}
+export interface Hypothesis {
+    utterance: string;
+    confidence: number;
+}
+export interface RecogniseParameters {
+    noInputTimeout?: number;
+    completeTimeout?: number;
+    locale?: string;
+    hints?: string[];
+}
+//# sourceMappingURL=types.d.ts.map

package/dist/types.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"types.d.ts","sourceRoot":"","sources":["../src/types.ts"],"names":[],"mappings":"AAAA,MAAM,WAAW,gBAAgB;IAC/B,QAAQ,EAAE,MAAM,CAAC;IACjB,GAAG,EAAE,MAAM,CAAC;CACb;AAED,MAAM,WAAW,QAAQ;IACvB,MAAM,EAAE,MAAM,CAAC;IACf,gBAAgB,EAAE,MAAM,GAAG,gBAAgB,CAAC;IAC5C,yBAAyB,EAAE,MAAM,CAAC;IAClC,wBAAwB,EAAE,MAAM,CAAC;IACjC,eAAe,EAAE,MAAM,CAAC;CACzB;AAED,MAAM,WAAW,MAAM;IACrB,SAAS,EAAE,MAAM,CAAC;IAClB,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,MAAM,CAAC,EAAE,cAAc,CAAC,MAAM,CAAC,CAAC;CACjC;AAED,MAAM,WAAW,UAAU;IACzB,SAAS,EAAE,MAAM,CAAC;IAClB,UAAU,EAAE,MAAM,CAAC;CACpB;AAED,MAAM,WAAW,mBAAmB;IAClC,cAAc,CAAC,EAAE,MAAM,CAAC;IACxB,eAAe,CAAC,EAAE,MAAM,CAAC;IACzB,MAAM,CAAC,EAAE,MAAM,CAAC;IAChB,KAAK,CAAC,EAAE,MAAM,EAAE,CAAC;CAClB"}

package/dist/types.js

@@ -0,0 +1,2 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });

package/package.json

@@ -0,0 +1,57 @@
+{
+  "name": "speechstate",
+  "version": "2.0.0-beta.0",
+  "license": "GPL-3.0",
+  "homepage": "http://localhost/speechstate",
+  "main": "./dist/index.js",
+  "types": "./dist/index.d.ts",
+  "files": ["/dist"],
+  "dependencies": {
+    "microsoft-cognitiveservices-speech-sdk": "^1.31.0",
+    "web-speech-cognitive-services": "^7.1.2",
+    "xstate": "^5.5.1"
+  },
+  "scripts": {
+    "dev": "vite",
+    "test": "jest --silent=false"
+  },
+  "eslintConfig": {
+    "extends": ["react-app", "react-app/jest"]
+  },
+  "browserslist": {
+    "production": [">0.2%", "not dead", "not op_mini all"],
+    "development": [
+      "last 1 chrome version",
+      "last 1 firefox version",
+      "last 1 safari version"
+    ]
+  },
+  "devDependencies": {
+    "@mermaid-js/mermaid-cli": "^10.6.1",
+    "@parcel/transformer-typescript-types": "2.9.3",
+    "@types/jest": "^29.5.2",
+    "@types/webspeechapi": "^0.0.29",
+    "buffer": "^5.5.0||^6.0.0",
+    "jest": "^29.5.0",
+    "jest-environment-jsdom": "^29.5.0",
+    "parcel-reporter-static-files-copy": "^1.5.2",
+    "process": "^0.11.10",
+    "ts-jest": "^29.1.0",
+    "typescript": "^5.0.4",
+    "vite": "^5.0.11"
+  },
+  "packageManager": "yarn@3.6.4",
+  "description": "* SDK",
+  "directories": {
+    "doc": "docs",
+    "test": "test"
+  },
+  "repository": {
+    "type": "git",
+    "url": "git+ssh://git@github.com/vladmaraev/speechstate.git"
+  },
+  "author": "Vlad Maraev",
+  "bugs": {
+    "url": "https://github.com/vladmaraev/speechstate/issues"
+  }
+}