speechstate 2.0.0-beta.0

package/README.org

@@ -0,0 +1,92 @@
+#+TITLE: SpeechState
+
+* SDK
+
+** Spawn SpeechState
+#+begin_src typescript
+  import { speechstate } from "speechstate";
+
+  // start with this action in your state chart
+  assign({
+      spstRef: ({ spawn }) => {
+          return spawn(speechstate, {
+              input: {
+                  settings: ...
+              },
+          });
+      },
+  }),
+#+end_src
+
+#+begin_src typescript
+interface AzureCredentials {
+  endpoint: string;
+  key: string;
+}
+
+interface Settings {
+  locale: string;
+  azureCredentials: string | AzureCredentials; // you can use proxy URL or full credentials
+  asrDefaultCompleteTimeout: number;
+}
+#+end_src
+
+
+** Events
+Example action:
+#+begin_src typescript
+  ({ context }) =>
+    context.spstRef.send({
+      type: "SPEAK",
+      value: { utterance: "Hello world", voice: "en-GB-RyanNeural" },
+    });
+#+end_src
+
+*** DM to SpeechState
+- ~{ type: "PREPARE" }~  
+- ~{ type: "SPEAK"; value: Agenda }~ 
+- ~{ type: "LISTEN" }~
+- ~{ type: "CONTROL" }~
+- ~{ type: "STOP" }~
+
+
+*** SpeechState to DM
+- ~{ type: "ASRTTS_READY" }~  
+- ~{ type: "ASR_NOINPUT" }~
+- ~{ type: "RECOGNISED"; value: Hypothesis[] }~  
+- ~{ type: "SPEAK_COMPLETE" }~ 
+- ~{ type: "ASR_STARTED" }~ 
+- ~{ type: "TTS_STARTED" }~ 
+** Types:
+#+begin_src typescript
+interface Hypothesis {
+    utterance: string;
+    confidence: number;
+}
+  
+interface Agenda {
+  utterance: string;
+  voice?: string; // defaults to "en-US-DavisNeural"
+  streamURL?: string;
+}
+#+end_src
+
+
+* How to run ~SpeechState~
+** Create Azure account and enable speech services
+1. Apply for free student credits
+   https://azure.microsoft.com/en-us/free/students/. You should be
+   able to login with your GU account.
+2. Make sure that you are logged into the Azure portal (https://portal.azure.com/).
+3. Create a *Resource group* (you can use search field):
+   - Subscription: *Azure for students*
+   - Resource group: any name
+   - Region: *(Europe) North Europe*
+4. Create a *Speech service*:
+   - Name: any name
+   - Subscription: *Azure for students*
+   - Location: *(Europe) North Europe*
+   - Pricing tier: *Free (F0)*
+   - Resource group: group name from the previous step
+5. Within your Speech Service go to: *Resourse management → Keys and
+   Endpoint* and copy your KEY 1.

package/dist/asr.d.ts

@@ -0,0 +1,54 @@
+/// <reference types="webspeechapi" />
+import { RecogniseParameters, Hypothesis, AzureCredentials } from "./types";
+interface MySpeechRecognition extends SpeechRecognition {
+    new (): any;
+}
+interface MySpeechGrammarList extends SpeechGrammarList {
+    new (): any;
+}
+type ASREvent = {
+    type: "READY";
+    value: {
+        wsaASR: MySpeechRecognition;
+        wsaGrammarList: MySpeechGrammarList;
+    };
+} | {
+    type: "ERROR";
+} | {
+    type: "NOINPUT";
+} | {
+    type: "CONTROL";
+} | {
+    type: "START";
+    value?: RecogniseParameters;
+} | {
+    type: "STARTED";
+    value: {
+        wsaASRinstance: MySpeechRecognition;
+    };
+} | {
+    type: "STARTSPEECH";
+} | {
+    type: "RECOGNISED";
+} | {
+    type: "RESULT";
+    value: Hypothesis[];
+};
+interface ASRContext extends ASRInit {
+    azureAuthorizationToken?: string;
+    wsaASR?: MySpeechRecognition;
+    wsaASRinstance?: MySpeechRecognition;
+    wsaGrammarList?: MySpeechGrammarList;
+    result?: Hypothesis[];
+    params?: RecogniseParameters;
+}
+interface ASRInit {
+    asrDefaultCompleteTimeout: number;
+    asrDefaultNoInputTimeout: number;
+    locale: string;
+    audioContext: AudioContext;
+    azureCredentials: string | AzureCredentials;
+}
+export declare const asrMachine: import("xstate").StateMachine<ASRContext, ASREvent, Record<string, import("xstate").AnyActorRef>, import("xstate").ProvidedActor, import("xstate").ParameterizedObject, import("xstate").ParameterizedObject, string, import("xstate").StateValue, string, ASRInit, {}, import("xstate").ResolveTypegenMeta<import("xstate").TypegenDisabled, ASREvent, import("xstate").ProvidedActor, import("xstate").ParameterizedObject, import("xstate").ParameterizedObject, string, string>>;
+export {};
+//# sourceMappingURL=asr.d.ts.map

package/dist/asr.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"asr.d.ts","sourceRoot":"","sources":["../src/asr.ts"],"names":[],"mappings":";AAaA,OAAO,EAAE,mBAAmB,EAAE,UAAU,EAAE,gBAAgB,EAAE,MAAM,SAAS,CAAC;AAC5E,UAAU,mBAAoB,SAAQ,iBAAiB;IACrD,YAAO;CACR;AAED,UAAU,mBAAoB,SAAQ,iBAAiB;IACrD,YAAO;CACR;AAED,KAAK,QAAQ,GACT;IACE,IAAI,EAAE,OAAO,CAAC;IACd,KAAK,EAAE;QACL,MAAM,EAAE,mBAAmB,CAAC;QAC5B,cAAc,EAAE,mBAAmB,CAAC;KACrC,CAAC;CACH,GACD;IAAE,IAAI,EAAE,OAAO,CAAA;CAAE,GACjB;IAAE,IAAI,EAAE,SAAS,CAAA;CAAE,GACnB;IAAE,IAAI,EAAE,SAAS,CAAA;CAAE,GACnB;IACE,IAAI,EAAE,OAAO,CAAC;IACd,KAAK,CAAC,EAAE,mBAAmB,CAAC;CAC7B,GACD;IAAE,IAAI,EAAE,SAAS,CAAC;IAAC,KAAK,EAAE;QAAE,cAAc,EAAE,mBAAmB,CAAA;KAAE,CAAA;CAAE,GACnE;IAAE,IAAI,EAAE,aAAa,CAAA;CAAE,GACvB;IAAE,IAAI,EAAE,YAAY,CAAA;CAAE,GACtB;IAAE,IAAI,EAAE,QAAQ,CAAC;IAAC,KAAK,EAAE,UAAU,EAAE,CAAA;CAAE,CAAC;AAE5C,UAAU,UAAW,SAAQ,OAAO;IAClC,uBAAuB,CAAC,EAAE,MAAM,CAAC;IACjC,MAAM,CAAC,EAAE,mBAAmB,CAAC;IAC7B,cAAc,CAAC,EAAE,mBAAmB,CAAC;IACrC,cAAc,CAAC,EAAE,mBAAmB,CAAC;IACrC,MAAM,CAAC,EAAE,UAAU,EAAE,CAAC;IACtB,MAAM,CAAC,EAAE,mBAAmB,CAAC;CAC9B;AAED,UAAU,OAAO;IACf,yBAAyB,EAAE,MAAM,CAAC;IAClC,wBAAwB,EAAE,MAAM,CAAC;IACjC,MAAM,EAAE,MAAM,CAAC;IACf,YAAY,EAAE,YAAY,CAAC;IAC3B,gBAAgB,EAAE,MAAM,GAAG,gBAAgB,CAAC;CAC7C;AAED,eAAO,MAAM,UAAU,sdAyQtB,CAAC"}

package/dist/asr.js

@@ -0,0 +1,247 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.asrMachine = void 0;
+const xstate_1 = require("xstate");
+const getToken_1 = require("./getToken");
+const SpeechToText_1 = require("web-speech-cognitive-services/lib/SpeechServices/SpeechToText");
+const REGION = "northeurope";
+exports.asrMachine = (0, xstate_1.createMachine)({
+    id: "asr",
+    types: {},
+    context: ({ input }) => ({
+        asrDefaultCompleteTimeout: input.asrDefaultCompleteTimeout || 0,
+        asrDefaultNoInputTimeout: input.asrDefaultNoInputTimeout || 5000,
+        locale: input.locale || "en-US",
+        audioContext: input.audioContext,
+        azureCredentials: input.azureCredentials,
+    }),
+    initial: "GetToken",
+    on: {
+        READY: {
+            target: ".Ready",
+            actions: [
+                (0, xstate_1.assign)({
+                    wsaASR: ({ event }) => event.value.wsaASR,
+                    wsaGrammarList: ({ event }) => event.value.wsaGrammarList,
+                }),
+                (0, xstate_1.sendParent)({ type: "ASR_READY" }),
+            ],
+        },
+    },
+    states: {
+        Fail: {},
+        Ready: {
+            on: {
+                START: {
+                    target: "Recognising",
+                    actions: (0, xstate_1.assign)({ params: ({ event }) => event.value }),
+                },
+            },
+        },
+        Recognising: {
+            initial: "WaitForRecogniser",
+            invoke: {
+                id: "recStart",
+                input: ({ context }) => ({
+                    wsaASR: context.wsaASR,
+                    wsaGrammarList: context.wsaGrammarList,
+                    locale: context.locale,
+                    phrases: (context.params || {}).hints || [],
+                }),
+                src: "recStart",
+            },
+            exit: "recStop",
+            on: {
+                RESULT: {
+                    actions: [
+                        (0, xstate_1.assign)({
+                            result: ({ event }) => event.value,
+                        }),
+                        (0, xstate_1.cancel)("completeTimeout"),
+                    ],
+                    target: ".Match",
+                },
+                RECOGNISED: {
+                    target: "Ready",
+                    actions: [
+                        (0, xstate_1.sendParent)(({ context }) => ({
+                            type: "RECOGNISED",
+                            value: context.result,
+                        })),
+                    ],
+                },
+                CONTROL: {
+                    target: "Pause",
+                },
+                NOINPUT: {
+                    actions: (0, xstate_1.sendParent)({ type: "ASR_NOINPUT" }),
+                    target: "Ready",
+                },
+            },
+            states: {
+                WaitForRecogniser: {
+                    on: {
+                        STARTED: {
+                            target: "NoInput",
+                            actions: [
+                                (0, xstate_1.assign)({
+                                    wsaASRinstance: ({ event }) => event.value.wsaASRinstance,
+                                }),
+                                (0, xstate_1.sendParent)({ type: "ASR_STARTED" }),
+                            ],
+                        },
+                    },
+                },
+                NoInput: {
+                    entry: [
+                        (0, xstate_1.raise)({ type: "NOINPUT" }, {
+                            delay: ({ context }) => (context.params || {}).noInputTimeout ||
+                                context.asrDefaultNoInputTimeout,
+                            id: "timeout",
+                        }),
+                    ],
+                    on: {
+                        STARTSPEECH: {
+                            target: "InProgress",
+                            actions: (0, xstate_1.cancel)("completeTimeout"),
+                        },
+                    },
+                    exit: [(0, xstate_1.cancel)("timeout")],
+                },
+                InProgress: {
+                    entry: () => console.debug("[ASR] in progress"),
+                },
+                Match: {
+                    entry: [
+                        ({ context }) => console.debug("RECOGNISED will be sent in (ms)", (context.params || {}).completeTimeout ||
+                            context.asrDefaultCompleteTimeout),
+                        (0, xstate_1.raise)({ type: "RECOGNISED" }, {
+                            delay: ({ context }) => (context.params || {}).completeTimeout ||
+                                context.asrDefaultCompleteTimeout,
+                            id: "completeTimeout",
+                        }),
+                    ],
+                },
+            },
+        },
+        Pause: {
+            entry: (0, xstate_1.sendParent)({ type: "ASR_PAUSED" }),
+            on: {
+                CONTROL: {
+                    target: "Recognising",
+                    //       ///// todo? reset noInputTimeout
+                    //       // actions: assign({
+                    //       //   params: {
+                    //       //     noInputTimeout: ({ context }) =>
+                    //       //       context.asrDefaultNoInputTimeout,
+                    //       //     completeTimeout: 0,
+                    //       //     locale: "0",
+                    //       //     hints: [""],
+                    //       //   },
+                    //       // }),} },
+                },
+            },
+        },
+        GetToken: {
+            invoke: {
+                id: "getAuthorizationToken",
+                input: ({ context }) => ({
+                    credentials: context.azureCredentials,
+                }),
+                src: "getToken",
+                onDone: {
+                    target: "Ponyfill",
+                    actions: [
+                        (0, xstate_1.assign)(({ event }) => {
+                            return { azureAuthorizationToken: event.output };
+                        }),
+                    ],
+                },
+                onError: {
+                    target: "Fail",
+                },
+            },
+        },
+        Ponyfill: {
+            invoke: {
+                id: "ponyASR",
+                src: "ponyfill",
+                input: ({ context }) => ({
+                    audioContext: context.audioContext,
+                    azureAuthorizationToken: context.azureAuthorizationToken,
+                    locale: context.locale,
+                }),
+            },
+        },
+    },
+}, {
+    actions: {
+        recStop: ({ context }) => {
+            context.wsaASRinstance.abort();
+            console.debug("[ASR] stopped");
+        },
+    },
+    actors: {
+        getToken: getToken_1.getToken,
+        ponyfill: (0, xstate_1.fromCallback)(({ sendBack, input }) => {
+            const { SpeechGrammarList, SpeechRecognition } = (0, SpeechToText_1.default)({
+                audioContext: input.audioContext,
+                credentials: {
+                    region: REGION,
+                    authorizationToken: input.azureAuthorizationToken,
+                },
+            });
+            sendBack({
+                type: "READY",
+                value: {
+                    wsaASR: SpeechRecognition,
+                    wsaGrammarList: SpeechGrammarList,
+                },
+            });
+            console.debug("[ASR] READY");
+        }),
+        recStart: (0, xstate_1.fromCallback)(({ sendBack, input }) => {
+            let asr = new input.wsaASR();
+            asr.grammars = new input.wsaGrammarList();
+            asr.grammars.phrases = input.phrases || [];
+            asr.lang = input.locale;
+            asr.continuous = true;
+            asr.interimResults = true;
+            asr.onresult = function (event) {
+                if (event.results[event.results.length - 1].isFinal) {
+                    const transcript = event.results
+                        .map((x) => x[0].transcript.replace(/\.$/, ""))
+                        .join(" ");
+                    const confidence = event.results
+                        .map((x) => x[0].confidence)
+                        .reduce((a, b) => a + b) /
+                        event.results.length;
+                    const res = [
+                        {
+                            utterance: transcript,
+                            confidence: confidence,
+                        },
+                    ];
+                    sendBack({
+                        type: "RESULT",
+                        value: res,
+                    });
+                    console.debug("[ASR] RESULT (pre-final)", res);
+                }
+                else {
+                    sendBack({ type: "STARTSPEECH" });
+                }
+            };
+            asr.addEventListener("start", () => {
+                sendBack({ type: "STARTED", value: { wsaASRinstance: asr } });
+            });
+            // receive((event) => {
+            //   console.debug("bla");
+            //   if (event.type === "STOP") {
+            //     asr.abort();
+            //   }
+            // });
+            asr.start();
+        }),
+    },
+});

package/dist/getToken.d.ts

@@ -0,0 +1,2 @@
+export declare const getToken: import("xstate").PromiseActorLogic<string, any>;
+//# sourceMappingURL=getToken.d.ts.map

package/dist/getToken.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"getToken.d.ts","sourceRoot":"","sources":["../src/getToken.ts"],"names":[],"mappings":"AAEA,eAAO,MAAM,QAAQ,iDAanB,CAAC"}

package/dist/getToken.js

@@ -0,0 +1,26 @@
+"use strict";
+var __awaiter = (this && this.__awaiter) || function (thisArg, _arguments, P, generator) {
+    function adopt(value) { return value instanceof P ? value : new P(function (resolve) { resolve(value); }); }
+    return new (P || (P = Promise))(function (resolve, reject) {
+        function fulfilled(value) { try { step(generator.next(value)); } catch (e) { reject(e); } }
+        function rejected(value) { try { step(generator["throw"](value)); } catch (e) { reject(e); } }
+        function step(result) { result.done ? resolve(result.value) : adopt(result.value).then(fulfilled, rejected); }
+        step((generator = generator.apply(thisArg, _arguments || [])).next());
+    });
+};
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.getToken = void 0;
+const xstate_1 = require("xstate");
+exports.getToken = (0, xstate_1.fromPromise)(({ input }) => __awaiter(void 0, void 0, void 0, function* () {
+    if (typeof input.credentials === "string") {
+        return fetch(new Request(input.credentials)).then((data) => data.text());
+    }
+    else {
+        return fetch(new Request(input.credentials.endpoint, {
+            method: "POST",
+            headers: {
+                "Ocp-Apim-Subscription-Key": input.credentials.key,
+            },
+        })).then((data) => data.text());
+    }
+}));

package/dist/index.d.ts

@@ -0,0 +1,3 @@
+export { speechstate } from "./speechstate";
+export { AzureCredentials, Settings, Hypothesis, Agenda, RecogniseParameters, } from "./types";
+//# sourceMappingURL=index.d.ts.map

package/dist/index.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,WAAW,EAAE,MAAM,eAAe,CAAC;AAC5C,OAAO,EACL,gBAAgB,EAChB,QAAQ,EACR,UAAU,EACV,MAAM,EACN,mBAAmB,GACpB,MAAM,SAAS,CAAC"}

package/dist/index.js

@@ -0,0 +1,5 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.speechstate = void 0;
+var speechstate_1 = require("./speechstate");
+Object.defineProperty(exports, "speechstate", { enumerable: true, get: function () { return speechstate_1.speechstate; } });

package/dist/speechstate.d.ts

@@ -0,0 +1,39 @@
+import { Settings, Agenda, Hypothesis } from "./types";
+interface SSContext {
+    settings: Settings;
+    audioContext?: AudioContext;
+    asrRef?: any;
+    ttsRef?: any;
+}
+type SSEvent = {
+    type: "PREPARE";
+} | {
+    type: "CONTROL";
+} | {
+    type: "STOP";
+} | {
+    type: "SPEAK";
+    value: Agenda;
+} | {
+    type: "TTS_READY";
+} | {
+    type: "TTS_STARTED";
+} | {
+    type: "TTS_ERROR";
+} | {
+    type: "SPEAK_COMPLETE";
+} | {
+    type: "ASR_READY";
+} | {
+    type: "LISTEN";
+} | {
+    type: "ASR_STARTED";
+} | {
+    type: "ASR_NOINPUT";
+} | {
+    type: "RECOGNISED";
+    value: Hypothesis[];
+};
+declare const speechstate: import("xstate").StateMachine<SSContext, SSEvent, Record<string, import("xstate").AnyActorRef>, import("xstate").ProvidedActor, import("xstate").ParameterizedObject, import("xstate").ParameterizedObject, string, import("xstate").StateValue, string, Settings, {}, import("xstate").ResolveTypegenMeta<import("xstate").TypegenDisabled, SSEvent, import("xstate").ProvidedActor, import("xstate").ParameterizedObject, import("xstate").ParameterizedObject, string, string>>;
+export { speechstate };
+//# sourceMappingURL=speechstate.d.ts.map

package/dist/speechstate.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"speechstate.d.ts","sourceRoot":"","sources":["../src/speechstate.ts"],"names":[],"mappings":"AAIA,OAAO,EAAE,QAAQ,EAAE,MAAM,EAAE,UAAU,EAAE,MAAM,SAAS,CAAC;AACvD,UAAU,SAAS;IACjB,QAAQ,EAAE,QAAQ,CAAC;IACnB,YAAY,CAAC,EAAE,YAAY,CAAC;IAC5B,MAAM,CAAC,EAAE,GAAG,CAAC;IACb,MAAM,CAAC,EAAE,GAAG,CAAC;CACd;AAED,KAAK,OAAO,GACR;IAAE,IAAI,EAAE,SAAS,CAAA;CAAE,GACnB;IAAE,IAAI,EAAE,SAAS,CAAA;CAAE,GACnB;IAAE,IAAI,EAAE,MAAM,CAAA;CAAE,GAChB;IAAE,IAAI,EAAE,OAAO,CAAC;IAAC,KAAK,EAAE,MAAM,CAAA;CAAE,GAChC;IAAE,IAAI,EAAE,WAAW,CAAA;CAAE,GACrB;IAAE,IAAI,EAAE,aAAa,CAAA;CAAE,GACvB;IAAE,IAAI,EAAE,WAAW,CAAA;CAAE,GACrB;IAAE,IAAI,EAAE,gBAAgB,CAAA;CAAE,GAC1B;IAAE,IAAI,EAAE,WAAW,CAAA;CAAE,GACrB;IAAE,IAAI,EAAE,QAAQ,CAAA;CAAE,GAClB;IAAE,IAAI,EAAE,aAAa,CAAA;CAAE,GACvB;IAAE,IAAI,EAAE,aAAa,CAAA;CAAE,GACvB;IAAE,IAAI,EAAE,YAAY,CAAC;IAAC,KAAK,EAAE,UAAU,EAAE,CAAA;CAAE,CAAC;AAEhD,QAAA,MAAM,WAAW,odAuPhB,CAAC;AAEF,OAAO,EAAE,WAAW,EAAE,CAAC"}