speechstate 2.0.0-beta.1 → 2.0.0-beta.10

package/README.org

@@ -1,3 +1,4 @@
+#+STARTUP: inlineimages
 #+TITLE: SpeechState
 
 * SDK
@@ -19,16 +20,20 @@
 #+end_src
 
 #+begin_src typescript
-interface AzureCredentials {
-  endpoint: string;
-  key: string;
-}
-
-interface Settings {
-  locale: string;
-  azureCredentials: string | AzureCredentials; // you can use proxy URL or full credentials
-  asrDefaultCompleteTimeout: number;
-}
+  interface AzureCredentials {
+    endpoint: string;
+    key: string;
+  }
+
+  interface Settings {
+    locale?: string;
+    azureCredentials: string | AzureCredentials;
+    azureLanguageCredentials?: AzureLanguageCredentials;
+    asrDefaultCompleteTimeout?: number;
+    asrDefaultNoInputTimeout?: number;
+    ttsDefaultVoice?: string;
+    speechRecognitionEndpointId?: string;
+  }
 #+end_src
 
 
@@ -45,7 +50,7 @@ Example action:
 *** DM to SpeechState
 - ~{ type: "PREPARE" }~  
 - ~{ type: "SPEAK"; value: Agenda }~ 
-- ~{ type: "LISTEN" }~
+- ~{ type: "LISTEN"; value?: RecogniseParameters }~
 - ~{ type: "CONTROL" }~
 - ~{ type: "STOP" }~
 
@@ -53,22 +58,38 @@ Example action:
 *** SpeechState to DM
 - ~{ type: "ASRTTS_READY" }~  
 - ~{ type: "ASR_NOINPUT" }~
-- ~{ type: "RECOGNISED"; value: Hypothesis[] }~  
+- ~{ type: "RECOGNISED"; value: Hypothesis[]; nluValue?: any }~  
 - ~{ type: "SPEAK_COMPLETE" }~ 
 - ~{ type: "ASR_STARTED" }~ 
 - ~{ type: "TTS_STARTED" }~ 
 ** Types:
 #+begin_src typescript
-interface Hypothesis {
+  interface Hypothesis {
+      utterance: string;
+      confidence: number;
+  }
+
+  interface Agenda {
     utterance: string;
-    confidence: number;
-}
-  
-interface Agenda {
-  utterance: string;
-  voice?: string; // defaults to "en-US-DavisNeural"
-  streamURL?: string;
-}
+    voice?: string; // defaults to "en-US-DavisNeural"
+    streamURL?: string;
+  }
+
+  interface RecogniseParameters {
+    noInputTimeout?: number;
+    completeTimeout?: number;
+    locale?: string;
+    hints?: string[];
+    nlu?: boolean | AzureLanguageCredentials;
+  }
+
+  interface AzureLanguageCredentials {
+    endpoint: string;
+    key: string;
+    projectName: string;
+    deploymentName: string;
+  }
+
 #+end_src
 
 
@@ -90,3 +111,33 @@ interface Agenda {
    - Resource group: group name from the previous step
 5. Within your Speech Service go to: *Resourse management → Keys and
    Endpoint* and copy your KEY 1.
+* Sequence diagrams
+#+begin_src plantuml :results output replace :file docs/diagrams/dm-speechstate.svg :exports results
+  skinparam defaultFontName Helvetica
+  participant       SpeechState       as SS
+  hnote across: Initialization
+  create SS
+  DM -> SS : **spawn**
+  DM -> SS : PREPARE
+  SS -> DM : ASRTTS_READY
+  hnote across: Speech Synthesis 
+  DM -> SS : SPEAK
+  activate SS
+  SS --> DM : TTS_STARTED
+  return SPEAK_COMPLETE
+  hnote across: Speech Recognition
+  DM -> SS : LISTEN
+  activate SS
+  SS --> DM : ASR_STARTED
+  return RECOGNISED
+  DM -> SS : LISTEN
+  activate SS
+  SS --> DM : ASR_STARTED
+  ... > noInputTimeout ...
+  return ASR_NOINPUT
+#+end_src
+
+#+RESULTS:
+[[file:docs/diagrams/dm-speechstate.svg]]
+
+

package/README.plantuml

@@ -0,0 +1,3 @@
+@startuml
+Bob -> Alice : Hello World!
+@enduml

package/dist/asr.d.ts

@@ -1,5 +1,5 @@
 /// <reference types="webspeechapi" />
-import { RecogniseParameters, Hypothesis, AzureCredentials } from "./types";
+import { RecogniseParameters, Hypothesis, AzureSpeechCredentials, AzureLanguageCredentials } from "./types";
 interface MySpeechRecognition extends SpeechRecognition {
     new (): any;
 }
@@ -40,6 +40,7 @@ interface ASRContext extends ASRInit {
     wsaASRinstance?: MySpeechRecognition;
     wsaGrammarList?: MySpeechGrammarList;
     result?: Hypothesis[];
+    nluResult?: any;
     params?: RecogniseParameters;
 }
 interface ASRInit {
@@ -47,8 +48,70 @@ interface ASRInit {
     asrDefaultNoInputTimeout: number;
     locale: string;
     audioContext: AudioContext;
-    azureCredentials: string | AzureCredentials;
+    azureCredentials: string | AzureSpeechCredentials;
+    speechRecognitionEndpointId?: string;
+    azureLanguageCredentials?: AzureLanguageCredentials;
 }
-export declare const asrMachine: import("xstate").StateMachine<ASRContext, ASREvent, Record<string, import("xstate").AnyActorRef>, import("xstate").ProvidedActor, import("xstate").ParameterizedObject, import("xstate").ParameterizedObject, string, import("xstate").StateValue, string, ASRInit, {}, import("xstate").ResolveTypegenMeta<import("xstate").TypegenDisabled, ASREvent, import("xstate").ProvidedActor, import("xstate").ParameterizedObject, import("xstate").ParameterizedObject, string, string>>;
+export declare const asrMachine: import("xstate").StateMachine<ASRContext, ASREvent, {
+    [x: string]: import("xstate").ActorRef<import("xstate").CallbackSnapshot<{}>, import("xstate").EventObject> | import("xstate").ActorRef<import("xstate").PromiseSnapshot<string, any>, {
+        [k: string]: unknown;
+        type: string;
+    }> | import("xstate").ActorRef<import("xstate").CallbackSnapshot<any>, import("xstate").EventObject> | import("xstate").ActorRef<import("xstate").PromiseSnapshot<any, AzureLanguageCredentials & {
+        query: string;
+    }>, {
+        [k: string]: unknown;
+        type: string;
+    }>;
+}, {
+    src: "getToken";
+    logic: import("xstate").PromiseActorLogic<string, any>;
+    id: string;
+} | {
+    src: "ponyfill";
+    logic: import("xstate").CallbackActorLogic<import("xstate").EventObject, {}>;
+    id: string;
+} | {
+    src: "recStart";
+    logic: import("xstate").CallbackActorLogic<import("xstate").EventObject, any>;
+    id: string;
+} | {
+    src: "nluPromise";
+    logic: import("xstate").PromiseActorLogic<any, AzureLanguageCredentials & {
+        query: string;
+    }>;
+    id: string;
+}, {
+    type: "recStop";
+    params: unknown;
+}, {
+    type: string;
+    params: {};
+}, string, "GetToken" | "Ready" | "Fail" | "Ponyfill" | "Pause" | {
+    Recognising?: "WaitForRecogniser" | "NoInput" | "InProgress" | "Match" | "NLURequest";
+}, string, ASRInit, {}, import("xstate").ResolveTypegenMeta<import("xstate").TypegenDisabled, ASREvent, {
+    src: "getToken";
+    logic: import("xstate").PromiseActorLogic<string, any>;
+    id: string;
+} | {
+    src: "ponyfill";
+    logic: import("xstate").CallbackActorLogic<import("xstate").EventObject, {}>;
+    id: string;
+} | {
+    src: "recStart";
+    logic: import("xstate").CallbackActorLogic<import("xstate").EventObject, any>;
+    id: string;
+} | {
+    src: "nluPromise";
+    logic: import("xstate").PromiseActorLogic<any, AzureLanguageCredentials & {
+        query: string;
+    }>;
+    id: string;
+}, {
+    type: "recStop";
+    params: unknown;
+}, {
+    type: string;
+    params: {};
+}, string, string>>;
 export {};
 //# sourceMappingURL=asr.d.ts.map

package/dist/asr.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"asr.d.ts","sourceRoot":"","sources":["../src/asr.ts"],"names":[],"mappings":";AAaA,OAAO,EAAE,mBAAmB,EAAE,UAAU,EAAE,gBAAgB,EAAE,MAAM,SAAS,CAAC;AAC5E,UAAU,mBAAoB,SAAQ,iBAAiB;IACrD,YAAO;CACR;AAED,UAAU,mBAAoB,SAAQ,iBAAiB;IACrD,YAAO;CACR;AAED,KAAK,QAAQ,GACT;IACE,IAAI,EAAE,OAAO,CAAC;IACd,KAAK,EAAE;QACL,MAAM,EAAE,mBAAmB,CAAC;QAC5B,cAAc,EAAE,mBAAmB,CAAC;KACrC,CAAC;CACH,GACD;IAAE,IAAI,EAAE,OAAO,CAAA;CAAE,GACjB;IAAE,IAAI,EAAE,SAAS,CAAA;CAAE,GACnB;IAAE,IAAI,EAAE,SAAS,CAAA;CAAE,GACnB;IACE,IAAI,EAAE,OAAO,CAAC;IACd,KAAK,CAAC,EAAE,mBAAmB,CAAC;CAC7B,GACD;IAAE,IAAI,EAAE,SAAS,CAAC;IAAC,KAAK,EAAE;QAAE,cAAc,EAAE,mBAAmB,CAAA;KAAE,CAAA;CAAE,GACnE;IAAE,IAAI,EAAE,aAAa,CAAA;CAAE,GACvB;IAAE,IAAI,EAAE,YAAY,CAAA;CAAE,GACtB;IAAE,IAAI,EAAE,QAAQ,CAAC;IAAC,KAAK,EAAE,UAAU,EAAE,CAAA;CAAE,CAAC;AAE5C,UAAU,UAAW,SAAQ,OAAO;IAClC,uBAAuB,CAAC,EAAE,MAAM,CAAC;IACjC,MAAM,CAAC,EAAE,mBAAmB,CAAC;IAC7B,cAAc,CAAC,EAAE,mBAAmB,CAAC;IACrC,cAAc,CAAC,EAAE,mBAAmB,CAAC;IACrC,MAAM,CAAC,EAAE,UAAU,EAAE,CAAC;IACtB,MAAM,CAAC,EAAE,mBAAmB,CAAC;CAC9B;AAED,UAAU,OAAO;IACf,yBAAyB,EAAE,MAAM,CAAC;IAClC,wBAAwB,EAAE,MAAM,CAAC;IACjC,MAAM,EAAE,MAAM,CAAC;IACf,YAAY,EAAE,YAAY,CAAC;IAC3B,gBAAgB,EAAE,MAAM,GAAG,gBAAgB,CAAC;CAC7C;AAED,eAAO,MAAM,UAAU,sdAyQtB,CAAC"}
+{"version":3,"file":"asr.d.ts","sourceRoot":"","sources":["../src/asr.ts"],"names":[],"mappings":";AAcA,OAAO,EACL,mBAAmB,EACnB,UAAU,EACV,sBAAsB,EACtB,wBAAwB,EACzB,MAAM,SAAS,CAAC;AAEjB,UAAU,mBAAoB,SAAQ,iBAAiB;IACrD,YAAO;CACR;AACD,UAAU,mBAAoB,SAAQ,iBAAiB;IACrD,YAAO;CACR;AAED,KAAK,QAAQ,GACT;IACE,IAAI,EAAE,OAAO,CAAC;IACd,KAAK,EAAE;QACL,MAAM,EAAE,mBAAmB,CAAC;QAC5B,cAAc,EAAE,mBAAmB,CAAC;KACrC,CAAC;CACH,GACD;IAAE,IAAI,EAAE,OAAO,CAAA;CAAE,GACjB;IAAE,IAAI,EAAE,SAAS,CAAA;CAAE,GACnB;IAAE,IAAI,EAAE,SAAS,CAAA;CAAE,GACnB;IACE,IAAI,EAAE,OAAO,CAAC;IACd,KAAK,CAAC,EAAE,mBAAmB,CAAC;CAC7B,GACD;IAAE,IAAI,EAAE,SAAS,CAAC;IAAC,KAAK,EAAE;QAAE,cAAc,EAAE,mBAAmB,CAAA;KAAE,CAAA;CAAE,GACnE;IAAE,IAAI,EAAE,aAAa,CAAA;CAAE,GACvB;IAAE,IAAI,EAAE,YAAY,CAAA;CAAE,GACtB;IAAE,IAAI,EAAE,QAAQ,CAAC;IAAC,KAAK,EAAE,UAAU,EAAE,CAAA;CAAE,CAAC;AAE5C,UAAU,UAAW,SAAQ,OAAO;IAClC,uBAAuB,CAAC,EAAE,MAAM,CAAC;IACjC,MAAM,CAAC,EAAE,mBAAmB,CAAC;IAC7B,cAAc,CAAC,EAAE,mBAAmB,CAAC;IACrC,cAAc,CAAC,EAAE,mBAAmB,CAAC;IACrC,MAAM,CAAC,EAAE,UAAU,EAAE,CAAC;IACtB,SAAS,CAAC,EAAE,GAAG,CAAC;IAChB,MAAM,CAAC,EAAE,mBAAmB,CAAC;CAC9B;AAED,UAAU,OAAO;IACf,yBAAyB,EAAE,MAAM,CAAC;IAClC,wBAAwB,EAAE,MAAM,CAAC;IACjC,MAAM,EAAE,MAAM,CAAC;IACf,YAAY,EAAE,YAAY,CAAC;IAC3B,gBAAgB,EAAE,MAAM,GAAG,sBAAsB,CAAC;IAClD,2BAA2B,CAAC,EAAE,MAAM,CAAC;IACrC,wBAAwB,CAAC,EAAE,wBAAwB,CAAC;CACrD;AAED,eAAO,MAAM,UAAU;;;;;eAiF8C,MAAM;;;;;;;;;;;;;;;;;;;;eAAN,MAAM;;;;;;;;;;;;;;;;;;;;;;;;;;eAAN,MAAM;;;;;;;;;mBA6RzE,CAAC"}

package/dist/asr.js

@@ -1,19 +1,133 @@
 "use strict";
+var __awaiter = (this && this.__awaiter) || function (thisArg, _arguments, P, generator) {
+    function adopt(value) { return value instanceof P ? value : new P(function (resolve) { resolve(value); }); }
+    return new (P || (P = Promise))(function (resolve, reject) {
+        function fulfilled(value) { try { step(generator.next(value)); } catch (e) { reject(e); } }
+        function rejected(value) { try { step(generator["throw"](value)); } catch (e) { reject(e); } }
+        function step(result) { result.done ? resolve(result.value) : adopt(result.value).then(fulfilled, rejected); }
+        step((generator = generator.apply(thisArg, _arguments || [])).next());
+    });
+};
 Object.defineProperty(exports, "__esModule", { value: true });
 exports.asrMachine = void 0;
 const xstate_1 = require("xstate");
 const getToken_1 = require("./getToken");
 const SpeechToText_1 = require("web-speech-cognitive-services/lib/SpeechServices/SpeechToText");
 const REGION = "northeurope";
-exports.asrMachine = (0, xstate_1.createMachine)({
+exports.asrMachine = (0, xstate_1.setup)({
+    types: {
+        context: {},
+        events: {},
+        input: {},
+    },
+    actions: {
+        recStop: ({ context }) => {
+            context.wsaASRinstance.abort();
+            console.debug("[ASR] stopped");
+        },
+    },
+    actors: {
+        getToken: getToken_1.getToken,
+        ponyfill: (0, xstate_1.fromCallback)(({ sendBack, input }) => {
+            const { SpeechGrammarList, SpeechRecognition } = (0, SpeechToText_1.default)({
+                audioContext: input.audioContext,
+                speechRecognitionEndpointId: input
+                    .speechRecognitionEndpointId,
+                credentials: {
+                    region: REGION, // TODO
+                    authorizationToken: input.azureAuthorizationToken,
+                },
+            });
+            sendBack({
+                type: "READY",
+                value: {
+                    wsaASR: SpeechRecognition,
+                    wsaGrammarList: SpeechGrammarList,
+                },
+            });
+            console.debug("[ASR] READY", input);
+        }),
+        recStart: (0, xstate_1.fromCallback)(({ sendBack, input }) => {
+            let asr = new input.wsaASR();
+            asr.grammars = new input.wsaGrammarList();
+            asr.grammars.phrases = input.phrases || [];
+            asr.lang = input.locale;
+            asr.continuous = true;
+            asr.interimResults = true;
+            asr.onresult = function (event) {
+                if (event.results[event.results.length - 1].isFinal) {
+                    const transcript = event.results
+                        .map((x) => x[0].transcript.replace(/\.$/, ""))
+                        .join(" ");
+                    const confidence = event.results
+                        .map((x) => x[0].confidence)
+                        .reduce((a, b) => a + b) / event.results.length;
+                    const res = [
+                        {
+                            utterance: transcript,
+                            confidence: confidence,
+                        },
+                    ];
+                    sendBack({
+                        type: "RESULT",
+                        value: res,
+                    });
+                    console.debug("[ASR] RESULT (pre-final)", res);
+                }
+                else {
+                    sendBack({ type: "STARTSPEECH" });
+                }
+            };
+            asr.addEventListener("start", () => {
+                sendBack({ type: "STARTED", value: { wsaASRinstance: asr } });
+            });
+            // receive((event) => {
+            //   console.debug("bla");
+            //   if (event.type === "STOP") {
+            //     asr.abort();
+            //   }
+            // });
+            asr.start();
+        }),
+        nluPromise: (0, xstate_1.fromPromise)(({ input }) => __awaiter(void 0, void 0, void 0, function* () {
+            const response = yield fetch(new Request(input.endpoint, {
+                method: "POST",
+                headers: {
+                    "Ocp-Apim-Subscription-Key": input.key,
+                    "Content-Type": "application/json",
+                },
+                body: JSON.stringify({
+                    kind: "Conversation",
+                    analysisInput: {
+                        conversationItem: {
+                            id: "PARTICIPANT_ID_HERE",
+                            text: input.query,
+                            modality: "text",
+                            language: "en-US", // TODO
+                            participantId: "PARTICIPANT_ID_HERE",
+                        },
+                    },
+                    parameters: {
+                        projectName: input.projectName,
+                        verbose: true,
+                        deploymentName: input.deploymentName,
+                        stringIndexType: "TextElement_V8",
+                    },
+                }),
+            }));
+            return response.json();
+        })),
+    },
+}).createMachine({
     id: "asr",
-    types: {},
     context: ({ input }) => ({
         asrDefaultCompleteTimeout: input.asrDefaultCompleteTimeout || 0,
         asrDefaultNoInputTimeout: input.asrDefaultNoInputTimeout || 5000,
         locale: input.locale || "en-US",
         audioContext: input.audioContext,
         azureCredentials: input.azureCredentials,
+        azureLanguageCredentials: input.azureLanguageCredentials,
+        speechRecognitionEndpointId: input.speechRecognitionEndpointId,
     }),
     initial: "GetToken",
     on: {
@@ -34,7 +148,7 @@ exports.asrMachine = (0, xstate_1.createMachine)({
             on: {
                 START: {
                     target: "Recognising",
-                    actions: (0, xstate_1.assign)({ params: ({ event }) => event.value }),
+                    actions: (0, xstate_1.assign)({ params: ({ event }) => event.value || {} }),
                 },
             },
         },
@@ -61,18 +175,27 @@ exports.asrMachine = (0, xstate_1.createMachine)({
                     ],
                     target: ".Match",
                 },
-                RECOGNISED: {
-                    target: "Ready",
-                    actions: [
-                        (0, xstate_1.sendParent)(({ context }) => ({
-                            type: "RECOGNISED",
-                            value: context.result,
-                        })),
-                    ],
-                },
+                RECOGNISED: [
+                    {
+                        target: ".NLURequest",
+                        guard: ({ context }) => !!(context.params.nlu && context.azureLanguageCredentials),
+                    },
+                    {
+                        target: "Ready",
+                        actions: [
+                            (0, xstate_1.sendParent)(({ context }) => ({
+                                type: "RECOGNISED",
+                                value: context.result,
+                            })),
+                        ],
+                    },
+                ],
                 CONTROL: {
                     target: "Pause",
                 },
+                STOP: {
+                    target: "Ready",
+                },
                 NOINPUT: {
                     actions: (0, xstate_1.sendParent)({ type: "ASR_NOINPUT" }),
                     target: "Ready",
@@ -122,6 +245,58 @@ exports.asrMachine = (0, xstate_1.createMachine)({
                         }),
                     ],
                 },
+                NLURequest: {
+                    invoke: {
+                        src: "nluPromise",
+                        input: ({ context }) => {
+                            let c;
+                            typeof context.params.nlu === "boolean"
+                                ? (c = context.azureLanguageCredentials)
+                                : (c = context.params.nlu);
+                            return {
+                                endpoint: c.endpoint,
+                                key: c.key,
+                                projectName: c.projectName,
+                                deploymentName: c.deploymentName,
+                                query: context.result[0].utterance,
+                            };
+                        },
+                        onDone: [
+                            {
+                                actions: [
+                                    ({ event }) => console.error("[ASR] no NLU prediction", event.output),
+                                    (0, xstate_1.sendParent)(({ context }) => ({
+                                        type: "RECOGNISED",
+                                        value: context.result,
+                                    })),
+                                ],
+                                target: "#asr.Ready",
+                                guard: ({ event }) => !(event.output.result || {}).prediction,
+                            },
+                            {
+                                actions: [
+                                    ({ event }) => console.debug("[ASR] NLU result", event.output.result.prediction),
+                                    (0, xstate_1.sendParent)(({ context, event }) => ({
+                                        type: "RECOGNISED",
+                                        value: context.result,
+                                        nluValue: event.output.result.prediction,
+                                    })),
+                                ],
+                                target: "#asr.Ready",
+                            },
+                        ],
+                        onError: {
+                            actions: [
+                                ({ event }) => console.error("[ASR]", event.error),
+                                (0, xstate_1.sendParent)(({ context }) => ({
+                                    type: "RECOGNISED",
+                                    value: context.result,
+                                })),
+                            ],
+                            target: "#asr.Ready",
+                        },
+                    },
+                },
             },
         },
         Pause: {
@@ -158,6 +333,7 @@ exports.asrMachine = (0, xstate_1.createMachine)({
                     ],
                 },
                 onError: {
+                    actions: ({ event }) => console.error("[ASR]", event.error),
                     target: "Fail",
                 },
             },
@@ -170,78 +346,9 @@ exports.asrMachine = (0, xstate_1.createMachine)({
                     audioContext: context.audioContext,
                     azureAuthorizationToken: context.azureAuthorizationToken,
                     locale: context.locale,
+                    speechRecognitionEndpointId: context.speechRecognitionEndpointId,
                 }),
             },
         },
     },
-}, {
-    actions: {
-        recStop: ({ context }) => {
-            context.wsaASRinstance.abort();
-            console.debug("[ASR] stopped");
-        },
-    },
-    actors: {
-        getToken: getToken_1.getToken,
-        ponyfill: (0, xstate_1.fromCallback)(({ sendBack, input }) => {
-            const { SpeechGrammarList, SpeechRecognition } = (0, SpeechToText_1.default)({
-                audioContext: input.audioContext,
-                credentials: {
-                    region: REGION,
-                    authorizationToken: input.azureAuthorizationToken,
-                },
-            });
-            sendBack({
-                type: "READY",
-                value: {
-                    wsaASR: SpeechRecognition,
-                    wsaGrammarList: SpeechGrammarList,
-                },
-            });
-            console.debug("[ASR] READY");
-        }),
-        recStart: (0, xstate_1.fromCallback)(({ sendBack, input }) => {
-            let asr = new input.wsaASR();
-            asr.grammars = new input.wsaGrammarList();
-            asr.grammars.phrases = input.phrases || [];
-            asr.lang = input.locale;
-            asr.continuous = true;
-            asr.interimResults = true;
-            asr.onresult = function (event) {
-                if (event.results[event.results.length - 1].isFinal) {
-                    const transcript = event.results
-                        .map((x) => x[0].transcript.replace(/\.$/, ""))
-                        .join(" ");
-                    const confidence = event.results
-                        .map((x) => x[0].confidence)
-                        .reduce((a, b) => a + b) /
-                        event.results.length;
-                    const res = [
-                        {
-                            utterance: transcript,
-                            confidence: confidence,
-                        },
-                    ];
-                    sendBack({
-                        type: "RESULT",
-                        value: res,
-                    });
-                    console.debug("[ASR] RESULT (pre-final)", res);
-                }
-                else {
-                    sendBack({ type: "STARTSPEECH" });
-                }
-            };
-            asr.addEventListener("start", () => {
-                sendBack({ type: "STARTED", value: { wsaASRinstance: asr } });
-            });
-            // receive((event) => {
-            //   console.debug("bla");
-            //   if (event.type === "STOP") {
-            //     asr.abort();
-            //   }
-            // });
-            asr.start();
-        }),
-    },
 });

package/dist/speechstate.d.ts

@@ -1,11 +1,12 @@
-import { Settings, Agenda, Hypothesis } from "./types";
+import { Settings, Agenda, Hypothesis, RecogniseParameters } from "./types";
 interface SSContext {
     settings: Settings;
     audioContext?: AudioContext;
     asrRef?: any;
     ttsRef?: any;
 }
-type SSEvent = {
+/** events sent to the spawned `speechstate` machine **/
+type SSEventExtIn = {
     type: "PREPARE";
 } | {
     type: "CONTROL";
@@ -14,26 +15,34 @@ type SSEvent = {
 } | {
     type: "SPEAK";
     value: Agenda;
-} | {
-    type: "TTS_READY";
-} | {
-    type: "TTS_STARTED";
-} | {
-    type: "TTS_ERROR";
-} | {
-    type: "SPEAK_COMPLETE";
-} | {
-    type: "ASR_READY";
 } | {
     type: "LISTEN";
+    value: RecogniseParameters;
+};
+/** for sendParent, not type-checked */
+type SSEventExtOut = {
+    type: "ASR_NOINPUT";
+} | {
+    type: "ASRTTS_READY";
 } | {
     type: "ASR_STARTED";
 } | {
-    type: "ASR_NOINPUT";
+    type: "TTS_STARTED";
+} | {
+    type: "SPEAK_COMPLETE";
 } | {
     type: "RECOGNISED";
     value: Hypothesis[];
+    nluValue?: any;
+};
+type SSEventIntIn = {
+    type: "TTS_READY";
+} | {
+    type: "ASR_READY";
+} | {
+    type: "TTS_ERROR";
 };
+type SSEvent = SSEventIntIn | SSEventExtIn | SSEventExtOut;
 declare const speechstate: import("xstate").StateMachine<SSContext, SSEvent, Record<string, import("xstate").AnyActorRef>, import("xstate").ProvidedActor, import("xstate").ParameterizedObject, import("xstate").ParameterizedObject, string, import("xstate").StateValue, string, Settings, {}, import("xstate").ResolveTypegenMeta<import("xstate").TypegenDisabled, SSEvent, import("xstate").ProvidedActor, import("xstate").ParameterizedObject, import("xstate").ParameterizedObject, string, string>>;
 export { speechstate };
 //# sourceMappingURL=speechstate.d.ts.map

package/dist/speechstate.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"speechstate.d.ts","sourceRoot":"","sources":["../src/speechstate.ts"],"names":[],"mappings":"AAIA,OAAO,EAAE,QAAQ,EAAE,MAAM,EAAE,UAAU,EAAE,MAAM,SAAS,CAAC;AACvD,UAAU,SAAS;IACjB,QAAQ,EAAE,QAAQ,CAAC;IACnB,YAAY,CAAC,EAAE,YAAY,CAAC;IAC5B,MAAM,CAAC,EAAE,GAAG,CAAC;IACb,MAAM,CAAC,EAAE,GAAG,CAAC;CACd;AAED,KAAK,OAAO,GACR;IAAE,IAAI,EAAE,SAAS,CAAA;CAAE,GACnB;IAAE,IAAI,EAAE,SAAS,CAAA;CAAE,GACnB;IAAE,IAAI,EAAE,MAAM,CAAA;CAAE,GAChB;IAAE,IAAI,EAAE,OAAO,CAAC;IAAC,KAAK,EAAE,MAAM,CAAA;CAAE,GAChC;IAAE,IAAI,EAAE,WAAW,CAAA;CAAE,GACrB;IAAE,IAAI,EAAE,aAAa,CAAA;CAAE,GACvB;IAAE,IAAI,EAAE,WAAW,CAAA;CAAE,GACrB;IAAE,IAAI,EAAE,gBAAgB,CAAA;CAAE,GAC1B;IAAE,IAAI,EAAE,WAAW,CAAA;CAAE,GACrB;IAAE,IAAI,EAAE,QAAQ,CAAA;CAAE,GAClB;IAAE,IAAI,EAAE,aAAa,CAAA;CAAE,GACvB;IAAE,IAAI,EAAE,aAAa,CAAA;CAAE,GACvB;IAAE,IAAI,EAAE,YAAY,CAAC;IAAC,KAAK,EAAE,UAAU,EAAE,CAAA;CAAE,CAAC;AAEhD,QAAA,MAAM,WAAW,odAuPhB,CAAC;AAEF,OAAO,EAAE,WAAW,EAAE,CAAC"}
+{"version":3,"file":"speechstate.d.ts","sourceRoot":"","sources":["../src/speechstate.ts"],"names":[],"mappings":"AAIA,OAAO,EAAE,QAAQ,EAAE,MAAM,EAAE,UAAU,EAAE,mBAAmB,EAAE,MAAM,SAAS,CAAC;AAC5E,UAAU,SAAS;IACjB,QAAQ,EAAE,QAAQ,CAAC;IACnB,YAAY,CAAC,EAAE,YAAY,CAAC;IAC5B,MAAM,CAAC,EAAE,GAAG,CAAC;IACb,MAAM,CAAC,EAAE,GAAG,CAAC;CACd;AAED,wDAAwD;AACxD,KAAK,YAAY,GACb;IAAE,IAAI,EAAE,SAAS,CAAA;CAAE,GACnB;IAAE,IAAI,EAAE,SAAS,CAAA;CAAE,GACnB;IAAE,IAAI,EAAE,MAAM,CAAA;CAAE,GAChB;IAAE,IAAI,EAAE,OAAO,CAAC;IAAC,KAAK,EAAE,MAAM,CAAA;CAAE,GAChC;IAAE,IAAI,EAAE,QAAQ,CAAC;IAAC,KAAK,EAAE,mBAAmB,CAAA;CAAE,CAAC;AAEnD,uCAAuC;AACvC,KAAK,aAAa,GACd;IAAE,IAAI,EAAE,aAAa,CAAA;CAAE,GACvB;IAAE,IAAI,EAAE,cAAc,CAAA;CAAE,GACxB;IAAE,IAAI,EAAE,aAAa,CAAA;CAAE,GACvB;IAAE,IAAI,EAAE,aAAa,CAAA;CAAE,GACvB;IAAE,IAAI,EAAE,gBAAgB,CAAA;CAAE,GAC1B;IAAE,IAAI,EAAE,YAAY,CAAC;IAAC,KAAK,EAAE,UAAU,EAAE,CAAC;IAAC,QAAQ,CAAC,EAAE,GAAG,CAAA;CAAE,CAAC;AAEhE,KAAK,YAAY,GACb;IAAE,IAAI,EAAE,WAAW,CAAA;CAAE,GACrB;IAAE,IAAI,EAAE,WAAW,CAAA;CAAE,GACrB;IAAE,IAAI,EAAE,WAAW,CAAA;CAAE,CAAC;AAE1B,KAAK,OAAO,GAAG,YAAY,GAAG,YAAY,GAAG,aAAa,CAAC;AAE3D,QAAA,MAAM,WAAW,odAuQhB,CAAC;AAEF,OAAO,EAAE,WAAW,EAAE,CAAC"}

package/dist/speechstate.js

@@ -12,21 +12,21 @@ const speechstate = (0, xstate_1.createMachine)({
     id: "speechstate",
     type: "parallel",
     states: {
-        asrTtsSpawner: {
-            initial: "idle",
+        AsrTtsSpawner: {
+            initial: "Idle",
             states: {
-                idle: { on: { PREPARE: "createAudioContext" } },
-                createAudioContext: {
+                Idle: { on: { PREPARE: "CreateAudioContext" } },
+                CreateAudioContext: {
                     invoke: {
                         id: "createAudioContext",
                         src: "audioContext",
                         onDone: {
-                            target: "spawn",
+                            target: "Spawn",
                             actions: (0, xstate_1.assign)({ audioContext: ({ event }) => event.output }),
                         },
                     },
                 },
-                spawn: {
+                Spawn: {
                     entry: [
                         (0, xstate_1.assign)({
                             ttsRef: ({ context, spawn }) => {
@@ -48,6 +48,8 @@ const speechstate = (0, xstate_1.createMachine)({
                                         locale: context.settings.locale,
                                         audioContext: context.audioContext,
                                         azureCredentials: context.settings.azureCredentials,
+                                        azureLanguageCredentials: context.settings.azureLanguageCredentials,
+                                        speechRecognitionEndpointId: context.settings.speechRecognitionEndpointId,
                                     },
                                 });
                             },
@@ -55,68 +57,68 @@ const speechstate = (0, xstate_1.createMachine)({
                     ],
                     after: {
                         300000: {
-                            target: "spawn",
+                            target: "Spawn",
                         },
                     },
                 },
             },
         },
-        asrTtsManager: {
-            initial: "initialize",
+        AsrTtsManager: {
+            initial: "Initialize",
             on: {
                 TTS_READY: {
                     actions: () => console.debug("[TTS→SpSt] TTS_READY"),
-                    target: ".preReady",
+                    target: ".PreReady",
                 },
                 ASR_READY: {
                     actions: () => console.debug("[ASR→SpSt] ASR_READY"),
-                    target: ".preReady",
+                    target: ".PreReady",
                 },
                 // ASR_ERROR not implemented
                 TTS_ERROR: {
                     actions: () => console.error("[TTS→SpSt] TTS_ERROR"),
-                    target: ".fail",
+                    target: ".Fail",
                 },
                 ASR_NOINPUT: {
                     actions: [
                         () => console.debug("[ASR→SpSt] NOINPUT"),
                         (0, xstate_1.sendParent)({ type: "ASR_NOINPUT" }),
                     ],
-                    target: ".ready",
+                    target: ".Ready",
                 },
             },
             states: {
-                initialize: {},
-                preReady: {
+                Initialize: {},
+                PreReady: {
                     on: {
                         TTS_READY: {
                             actions: () => console.debug("[TTS→SpSt] TTS_READY"),
-                            target: "ready",
+                            target: "Ready",
                         },
                         ASR_READY: {
                             actions: () => console.debug("[ASR→SpSt] ASR_READY"),
-                            target: "ready",
+                            target: "Ready",
                         },
                     },
                 },
-                ready: {
-                    initial: "idle",
+                Ready: {
+                    initial: "Idle",
                     entry: [
                         () => console.debug("[SpSt] All ready"),
                         (0, xstate_1.sendParent)({ type: "ASRTTS_READY" }),
                     ],
                     states: {
-                        idle: {
+                        Idle: {
                             on: {
-                                LISTEN: { target: "waitForRecogniser" },
+                                LISTEN: { target: "WaitForRecogniser" },
                                 SPEAK: [
                                     {
-                                        target: "speaking",
+                                        target: "Speaking",
                                     },
                                 ],
                             },
                         },
-                        speaking: {
+                        Speaking: {
                             entry: [
                                 ({ event }) => console.debug("[SpSt→TTS] SPEAK", event.value),
                                 ({ context, event }) => context.ttsRef.send({
@@ -148,7 +150,7 @@ const speechstate = (0, xstate_1.createMachine)({
                                     ],
                                 },
                                 SPEAK_COMPLETE: {
-                                    target: "idle",
+                                    target: "Idle",
                                     actions: [
                                         () => console.debug("[TTS→SpSt] SPEAK_COMPLETE"),
                                         (0, xstate_1.sendParent)({ type: "SPEAK_COMPLETE" }),
@@ -156,7 +158,7 @@ const speechstate = (0, xstate_1.createMachine)({
                                 },
                             },
                         },
-                        waitForRecogniser: {
+                        WaitForRecogniser: {
                             entry: [
                                 ({ event }) => console.debug("[SpSt→ASR] START", event.value),
                                 ({ context, event }) => context.asrRef.send({
@@ -166,7 +168,7 @@ const speechstate = (0, xstate_1.createMachine)({
                             ],
                             on: {
                                 ASR_STARTED: {
-                                    target: "recognising",
+                                    target: "Recognising",
                                     actions: [
                                         () => console.debug("[ASR→SpSt] ASR_STARTED"),
                                         (0, xstate_1.sendParent)({ type: "ASR_STARTED" }),
@@ -174,7 +176,7 @@ const speechstate = (0, xstate_1.createMachine)({
                                 },
                             },
                         },
-                        recognising: {
+                        Recognising: {
                             on: {
                                 CONTROL: {
                                     actions: [
@@ -184,21 +186,31 @@ const speechstate = (0, xstate_1.createMachine)({
                                         }),
                                     ],
                                 },
+                                STOP: {
+                                    target: "Idle",
+                                    actions: [
+                                        () => console.debug("[SpSt→ASR] STOP"),
+                                        ({ context }) => context.asrRef.send({
+                                            type: "STOP",
+                                        }),
+                                    ],
+                                },
                                 RECOGNISED: {
                                     actions: [
-                                        ({ event }) => console.debug("[ASR→SpSt] RECOGNISED", event.value),
+                                        ({ event }) => console.debug("[ASR→SpSt] RECOGNISED", event.value, event.nluValue),
                                         (0, xstate_1.sendParent)(({ event }) => ({
                                             type: "RECOGNISED",
                                             value: event.value,
+                                            nluValue: event.nluValue,
                                         })),
                                     ],
-                                    target: "idle",
+                                    target: "Idle",
                                 },
                             },
                         },
                     },
                 },
-                fail: {},
+                Fail: {},
             },
         },
     },

package/dist/tts.js

@@ -125,7 +125,7 @@ exports.ttsMachine = (0, xstate_1.createMachine)({
             const ponyfill = (0, TextToSpeech_1.default)({
                 audioContext: input.audioContext,
                 credentials: {
-                    region: REGION,
+                    region: REGION, // TODO
                     authorizationToken: input.azureAuthorizationToken,
                 },
             });

package/dist/types.d.ts

@@ -1,13 +1,26 @@
-export interface AzureCredentials {
+export interface AzureSpeechCredentials {
     endpoint: string;
     key: string;
 }
+/**
+ * @deprecated use `AzureSpeechCredentials` instead
+ */
+export interface AzureCredentials extends AzureSpeechCredentials {
+}
+export interface AzureLanguageCredentials {
+    endpoint: string;
+    key: string;
+    projectName: string;
+    deploymentName: string;
+}
 export interface Settings {
-    locale: string;
-    azureCredentials: string | AzureCredentials;
-    asrDefaultCompleteTimeout: number;
-    asrDefaultNoInputTimeout: number;
-    ttsDefaultVoice: string;
+    locale?: string;
+    azureCredentials: string | AzureCredentials | AzureSpeechCredentials;
+    azureLanguageCredentials?: AzureLanguageCredentials;
+    asrDefaultCompleteTimeout?: number;
+    asrDefaultNoInputTimeout?: number;
+    ttsDefaultVoice?: string;
+    speechRecognitionEndpointId?: string;
 }
 export interface Agenda {
     utterance: string;
@@ -23,5 +36,6 @@ export interface RecogniseParameters {
     completeTimeout?: number;
     locale?: string;
     hints?: string[];
+    nlu?: boolean | AzureLanguageCredentials;
 }
 //# sourceMappingURL=types.d.ts.map

package/dist/types.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"types.d.ts","sourceRoot":"","sources":["../src/types.ts"],"names":[],"mappings":"AAAA,MAAM,WAAW,gBAAgB;IAC/B,QAAQ,EAAE,MAAM,CAAC;IACjB,GAAG,EAAE,MAAM,CAAC;CACb;AAED,MAAM,WAAW,QAAQ;IACvB,MAAM,EAAE,MAAM,CAAC;IACf,gBAAgB,EAAE,MAAM,GAAG,gBAAgB,CAAC;IAC5C,yBAAyB,EAAE,MAAM,CAAC;IAClC,wBAAwB,EAAE,MAAM,CAAC;IACjC,eAAe,EAAE,MAAM,CAAC;CACzB;AAED,MAAM,WAAW,MAAM;IACrB,SAAS,EAAE,MAAM,CAAC;IAClB,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,MAAM,CAAC,EAAE,cAAc,CAAC,MAAM,CAAC,CAAC;CACjC;AAED,MAAM,WAAW,UAAU;IACzB,SAAS,EAAE,MAAM,CAAC;IAClB,UAAU,EAAE,MAAM,CAAC;CACpB;AAED,MAAM,WAAW,mBAAmB;IAClC,cAAc,CAAC,EAAE,MAAM,CAAC;IACxB,eAAe,CAAC,EAAE,MAAM,CAAC;IACzB,MAAM,CAAC,EAAE,MAAM,CAAC;IAChB,KAAK,CAAC,EAAE,MAAM,EAAE,CAAC;CAClB"}
+{"version":3,"file":"types.d.ts","sourceRoot":"","sources":["../src/types.ts"],"names":[],"mappings":"AAAA,MAAM,WAAW,sBAAsB;IACrC,QAAQ,EAAE,MAAM,CAAC;IACjB,GAAG,EAAE,MAAM,CAAC;CACb;AAED;;GAEG;AACH,MAAM,WAAW,gBAAiB,SAAQ,sBAAsB;CAAG;AAEnE,MAAM,WAAW,wBAAwB;IACvC,QAAQ,EAAE,MAAM,CAAC;IACjB,GAAG,EAAE,MAAM,CAAC;IACZ,WAAW,EAAE,MAAM,CAAC;IACpB,cAAc,EAAE,MAAM,CAAC;CACxB;AAED,MAAM,WAAW,QAAQ;IACvB,MAAM,CAAC,EAAE,MAAM,CAAC;IAChB,gBAAgB,EAAE,MAAM,GAAG,gBAAgB,GAAG,sBAAsB,CAAC;IACrE,wBAAwB,CAAC,EAAE,wBAAwB,CAAC;IACpD,yBAAyB,CAAC,EAAE,MAAM,CAAC;IACnC,wBAAwB,CAAC,EAAE,MAAM,CAAC;IAClC,eAAe,CAAC,EAAE,MAAM,CAAC;IACzB,2BAA2B,CAAC,EAAE,MAAM,CAAC;CACtC;AAED,MAAM,WAAW,MAAM;IACrB,SAAS,EAAE,MAAM,CAAC;IAClB,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,MAAM,CAAC,EAAE,cAAc,CAAC,MAAM,CAAC,CAAC;CACjC;AAED,MAAM,WAAW,UAAU;IACzB,SAAS,EAAE,MAAM,CAAC;IAClB,UAAU,EAAE,MAAM,CAAC;CACpB;AAED,MAAM,WAAW,mBAAmB;IAClC,cAAc,CAAC,EAAE,MAAM,CAAC;IACxB,eAAe,CAAC,EAAE,MAAM,CAAC;IACzB,MAAM,CAAC,EAAE,MAAM,CAAC;IAChB,KAAK,CAAC,EAAE,MAAM,EAAE,CAAC;IACjB,GAAG,CAAC,EAAE,OAAO,GAAG,wBAAwB,CAAC;CAC1C"}

package/package.json

@@ -1,18 +1,20 @@
 {
   "name": "speechstate",
-  "version": "2.0.0-beta.1",
+  "version": "2.0.0-beta.10",
   "license": "GPL-3.0",
   "homepage": "http://localhost/speechstate",
   "main": "./dist/index.js",
   "types": "./dist/index.d.ts",
   "files": ["/dist"],
   "dependencies": {
+    "@statelyai/inspect": "^0.2.3",
     "microsoft-cognitiveservices-speech-sdk": "^1.31.0",
-    "web-speech-cognitive-services": "^7.1.2",
+    "web-speech-cognitive-services": "^7.1.3",
     "xstate": "^5.5.1"
   },
   "scripts": {
     "dev": "vite",
+    "compile": "tsc",
     "test": "jest --silent=false"
   },
   "eslintConfig": {