speechstate 2.0.0-beta.5 → 2.0.0-beta.7
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.org +59 -11
- package/README.plantuml +3 -0
- package/README.png +0 -0
- package/dist/asr.d.ts +65 -3
- package/dist/asr.d.ts.map +1 -1
- package/dist/asr.js +184 -84
- package/dist/speechstate.d.ts +22 -13
- package/dist/speechstate.d.ts.map +1 -1
- package/dist/speechstate.js +3 -1
- package/dist/types.d.ts +15 -2
- package/dist/types.d.ts.map +1 -1
- package/package.json +3 -1
package/README.org
CHANGED
|
@@ -1,3 +1,4 @@
|
|
|
1
|
+
#+STARTUP: inlineimages
|
|
1
2
|
#+TITLE: SpeechState
|
|
2
3
|
|
|
3
4
|
* SDK
|
|
@@ -27,6 +28,7 @@
|
|
|
27
28
|
interface Settings {
|
|
28
29
|
locale?: string;
|
|
29
30
|
azureCredentials: string | AzureCredentials;
|
|
31
|
+
azureLanguageCredentials?: AzureLanguageCredentials;
|
|
30
32
|
asrDefaultCompleteTimeout?: number;
|
|
31
33
|
asrDefaultNoInputTimeout?: number;
|
|
32
34
|
ttsDefaultVoice?: string;
|
|
@@ -48,7 +50,7 @@ Example action:
|
|
|
48
50
|
*** DM to SpeechState
|
|
49
51
|
- ~{ type: "PREPARE" }~
|
|
50
52
|
- ~{ type: "SPEAK"; value: Agenda }~
|
|
51
|
-
- ~{ type: "LISTEN" }~
|
|
53
|
+
- ~{ type: "LISTEN"; value?: RecogniseParameters }~
|
|
52
54
|
- ~{ type: "CONTROL" }~
|
|
53
55
|
- ~{ type: "STOP" }~
|
|
54
56
|
|
|
@@ -56,22 +58,38 @@ Example action:
|
|
|
56
58
|
*** SpeechState to DM
|
|
57
59
|
- ~{ type: "ASRTTS_READY" }~
|
|
58
60
|
- ~{ type: "ASR_NOINPUT" }~
|
|
59
|
-
- ~{ type: "RECOGNISED"; value: Hypothesis[] }~
|
|
61
|
+
- ~{ type: "RECOGNISED"; value: Hypothesis[]; nluValue?: any }~
|
|
60
62
|
- ~{ type: "SPEAK_COMPLETE" }~
|
|
61
63
|
- ~{ type: "ASR_STARTED" }~
|
|
62
64
|
- ~{ type: "TTS_STARTED" }~
|
|
63
65
|
** Types:
|
|
64
66
|
#+begin_src typescript
|
|
65
|
-
interface Hypothesis {
|
|
67
|
+
interface Hypothesis {
|
|
68
|
+
utterance: string;
|
|
69
|
+
confidence: number;
|
|
70
|
+
}
|
|
71
|
+
|
|
72
|
+
interface Agenda {
|
|
66
73
|
utterance: string;
|
|
67
|
-
|
|
68
|
-
|
|
69
|
-
|
|
70
|
-
|
|
71
|
-
|
|
72
|
-
|
|
73
|
-
|
|
74
|
-
|
|
74
|
+
voice?: string; // defaults to "en-US-DavisNeural"
|
|
75
|
+
streamURL?: string;
|
|
76
|
+
}
|
|
77
|
+
|
|
78
|
+
interface RecogniseParameters {
|
|
79
|
+
noInputTimeout?: number;
|
|
80
|
+
completeTimeout?: number;
|
|
81
|
+
locale?: string;
|
|
82
|
+
hints?: string[];
|
|
83
|
+
nlu?: boolean | AzureLanguageCredentials;
|
|
84
|
+
}
|
|
85
|
+
|
|
86
|
+
interface AzureLanguageCredentials {
|
|
87
|
+
endpoint: string;
|
|
88
|
+
key: string;
|
|
89
|
+
projectName: string;
|
|
90
|
+
deploymentName: string;
|
|
91
|
+
}
|
|
92
|
+
|
|
75
93
|
#+end_src
|
|
76
94
|
|
|
77
95
|
|
|
@@ -93,3 +111,33 @@ interface Agenda {
|
|
|
93
111
|
- Resource group: group name from the previous step
|
|
94
112
|
5. Within your Speech Service go to: *Resourse management → Keys and
|
|
95
113
|
Endpoint* and copy your KEY 1.
|
|
114
|
+
* Sequence diagrams
|
|
115
|
+
#+begin_src plantuml :results output replace :file docs/diagrams/dm-speechstate.svg :exports results
|
|
116
|
+
skinparam defaultFontName Helvetica
|
|
117
|
+
participant SpeechState as SS
|
|
118
|
+
hnote across: Initialization
|
|
119
|
+
create SS
|
|
120
|
+
DM -> SS : **spawn**
|
|
121
|
+
DM -> SS : PREPARE
|
|
122
|
+
SS -> DM : ASRTTS_READY
|
|
123
|
+
hnote across: Speech Synthesis
|
|
124
|
+
DM -> SS : LISTEN
|
|
125
|
+
activate SS
|
|
126
|
+
SS --> DM : TTS_STARTED
|
|
127
|
+
return SPEAK_COMPLETE
|
|
128
|
+
hnote across: Speech Recognition
|
|
129
|
+
DM -> SS : LISTEN
|
|
130
|
+
activate SS
|
|
131
|
+
SS --> DM : ASR_STARTED
|
|
132
|
+
return RECOGNISED
|
|
133
|
+
DM -> SS : LISTEN
|
|
134
|
+
activate SS
|
|
135
|
+
SS --> DM : ASR_STARTED
|
|
136
|
+
... > noInputTimeout ...
|
|
137
|
+
return ASR_NOINPUT
|
|
138
|
+
#+end_src
|
|
139
|
+
|
|
140
|
+
#+RESULTS:
|
|
141
|
+
[[file:docs/diagrams/dm-speechstate.svg]]
|
|
142
|
+
|
|
143
|
+
|
package/README.plantuml
ADDED
package/README.png
ADDED
|
Binary file
|
package/dist/asr.d.ts
CHANGED
|
@@ -1,5 +1,5 @@
|
|
|
1
1
|
/// <reference types="webspeechapi" />
|
|
2
|
-
import { RecogniseParameters, Hypothesis,
|
|
2
|
+
import { RecogniseParameters, Hypothesis, AzureSpeechCredentials, AzureLanguageCredentials } from "./types";
|
|
3
3
|
interface MySpeechRecognition extends SpeechRecognition {
|
|
4
4
|
new (): any;
|
|
5
5
|
}
|
|
@@ -40,6 +40,7 @@ interface ASRContext extends ASRInit {
|
|
|
40
40
|
wsaASRinstance?: MySpeechRecognition;
|
|
41
41
|
wsaGrammarList?: MySpeechGrammarList;
|
|
42
42
|
result?: Hypothesis[];
|
|
43
|
+
nluResult?: any;
|
|
43
44
|
params?: RecogniseParameters;
|
|
44
45
|
}
|
|
45
46
|
interface ASRInit {
|
|
@@ -47,9 +48,70 @@ interface ASRInit {
|
|
|
47
48
|
asrDefaultNoInputTimeout: number;
|
|
48
49
|
locale: string;
|
|
49
50
|
audioContext: AudioContext;
|
|
50
|
-
azureCredentials: string |
|
|
51
|
+
azureCredentials: string | AzureSpeechCredentials;
|
|
51
52
|
speechRecognitionEndpointId?: string;
|
|
53
|
+
azureLanguageCredentials?: AzureLanguageCredentials;
|
|
52
54
|
}
|
|
53
|
-
export declare const asrMachine: import("xstate").StateMachine<ASRContext, ASREvent,
|
|
55
|
+
export declare const asrMachine: import("xstate").StateMachine<ASRContext, ASREvent, {
|
|
56
|
+
[x: string]: import("xstate").ActorRef<import("xstate").CallbackSnapshot<{}>, import("xstate").EventObject> | import("xstate").ActorRef<import("xstate").PromiseSnapshot<string, any>, {
|
|
57
|
+
[k: string]: unknown;
|
|
58
|
+
type: string;
|
|
59
|
+
}> | import("xstate").ActorRef<import("xstate").CallbackSnapshot<any>, import("xstate").EventObject> | import("xstate").ActorRef<import("xstate").PromiseSnapshot<any, AzureLanguageCredentials & {
|
|
60
|
+
query: string;
|
|
61
|
+
}>, {
|
|
62
|
+
[k: string]: unknown;
|
|
63
|
+
type: string;
|
|
64
|
+
}>;
|
|
65
|
+
}, {
|
|
66
|
+
src: "getToken";
|
|
67
|
+
logic: import("xstate").PromiseActorLogic<string, any>;
|
|
68
|
+
id: string;
|
|
69
|
+
} | {
|
|
70
|
+
src: "ponyfill";
|
|
71
|
+
logic: import("xstate").CallbackActorLogic<import("xstate").EventObject, {}>;
|
|
72
|
+
id: string;
|
|
73
|
+
} | {
|
|
74
|
+
src: "recStart";
|
|
75
|
+
logic: import("xstate").CallbackActorLogic<import("xstate").EventObject, any>;
|
|
76
|
+
id: string;
|
|
77
|
+
} | {
|
|
78
|
+
src: "nluPromise";
|
|
79
|
+
logic: import("xstate").PromiseActorLogic<any, AzureLanguageCredentials & {
|
|
80
|
+
query: string;
|
|
81
|
+
}>;
|
|
82
|
+
id: string;
|
|
83
|
+
}, {
|
|
84
|
+
type: "recStop";
|
|
85
|
+
params: unknown;
|
|
86
|
+
}, {
|
|
87
|
+
type: string;
|
|
88
|
+
params: {};
|
|
89
|
+
}, string, "GetToken" | "Ready" | "Fail" | "Ponyfill" | "Pause" | {
|
|
90
|
+
Recognising?: "WaitForRecogniser" | "NoInput" | "InProgress" | "Match" | "NLURequest";
|
|
91
|
+
}, string, ASRInit, {}, import("xstate").ResolveTypegenMeta<import("xstate").TypegenDisabled, ASREvent, {
|
|
92
|
+
src: "getToken";
|
|
93
|
+
logic: import("xstate").PromiseActorLogic<string, any>;
|
|
94
|
+
id: string;
|
|
95
|
+
} | {
|
|
96
|
+
src: "ponyfill";
|
|
97
|
+
logic: import("xstate").CallbackActorLogic<import("xstate").EventObject, {}>;
|
|
98
|
+
id: string;
|
|
99
|
+
} | {
|
|
100
|
+
src: "recStart";
|
|
101
|
+
logic: import("xstate").CallbackActorLogic<import("xstate").EventObject, any>;
|
|
102
|
+
id: string;
|
|
103
|
+
} | {
|
|
104
|
+
src: "nluPromise";
|
|
105
|
+
logic: import("xstate").PromiseActorLogic<any, AzureLanguageCredentials & {
|
|
106
|
+
query: string;
|
|
107
|
+
}>;
|
|
108
|
+
id: string;
|
|
109
|
+
}, {
|
|
110
|
+
type: "recStop";
|
|
111
|
+
params: unknown;
|
|
112
|
+
}, {
|
|
113
|
+
type: string;
|
|
114
|
+
params: {};
|
|
115
|
+
}, string, string>>;
|
|
54
116
|
export {};
|
|
55
117
|
//# sourceMappingURL=asr.d.ts.map
|
package/dist/asr.d.ts.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"asr.d.ts","sourceRoot":"","sources":["../src/asr.ts"],"names":[],"mappings":";
|
|
1
|
+
{"version":3,"file":"asr.d.ts","sourceRoot":"","sources":["../src/asr.ts"],"names":[],"mappings":";AAcA,OAAO,EACL,mBAAmB,EACnB,UAAU,EACV,sBAAsB,EACtB,wBAAwB,EACzB,MAAM,SAAS,CAAC;AAEjB,UAAU,mBAAoB,SAAQ,iBAAiB;IACrD,YAAO;CACR;AACD,UAAU,mBAAoB,SAAQ,iBAAiB;IACrD,YAAO;CACR;AAED,KAAK,QAAQ,GACT;IACE,IAAI,EAAE,OAAO,CAAC;IACd,KAAK,EAAE;QACL,MAAM,EAAE,mBAAmB,CAAC;QAC5B,cAAc,EAAE,mBAAmB,CAAC;KACrC,CAAC;CACH,GACD;IAAE,IAAI,EAAE,OAAO,CAAA;CAAE,GACjB;IAAE,IAAI,EAAE,SAAS,CAAA;CAAE,GACnB;IAAE,IAAI,EAAE,SAAS,CAAA;CAAE,GACnB;IACE,IAAI,EAAE,OAAO,CAAC;IACd,KAAK,CAAC,EAAE,mBAAmB,CAAC;CAC7B,GACD;IAAE,IAAI,EAAE,SAAS,CAAC;IAAC,KAAK,EAAE;QAAE,cAAc,EAAE,mBAAmB,CAAA;KAAE,CAAA;CAAE,GACnE;IAAE,IAAI,EAAE,aAAa,CAAA;CAAE,GACvB;IAAE,IAAI,EAAE,YAAY,CAAA;CAAE,GACtB;IAAE,IAAI,EAAE,QAAQ,CAAC;IAAC,KAAK,EAAE,UAAU,EAAE,CAAA;CAAE,CAAC;AAE5C,UAAU,UAAW,SAAQ,OAAO;IAClC,uBAAuB,CAAC,EAAE,MAAM,CAAC;IACjC,MAAM,CAAC,EAAE,mBAAmB,CAAC;IAC7B,cAAc,CAAC,EAAE,mBAAmB,CAAC;IACrC,cAAc,CAAC,EAAE,mBAAmB,CAAC;IACrC,MAAM,CAAC,EAAE,UAAU,EAAE,CAAC;IACtB,SAAS,CAAC,EAAE,GAAG,CAAC;IAChB,MAAM,CAAC,EAAE,mBAAmB,CAAC;CAC9B;AAED,UAAU,OAAO;IACf,yBAAyB,EAAE,MAAM,CAAC;IAClC,wBAAwB,EAAE,MAAM,CAAC;IACjC,MAAM,EAAE,MAAM,CAAC;IACf,YAAY,EAAE,YAAY,CAAC;IAC3B,gBAAgB,EAAE,MAAM,GAAG,sBAAsB,CAAC;IAClD,2BAA2B,CAAC,EAAE,MAAM,CAAC;IACrC,wBAAwB,CAAC,EAAE,wBAAwB,CAAC;CACrD;AAED,eAAO,MAAM,UAAU;;;;;eAiF8C,MAAM;;;;;;;;;;;;;;;;;;;;eAAN,MAAM;;;;;;;;;;;;;;;;;;;;;;;;;;eAAN,MAAM;;;;;;;;;mBA0RzE,CAAC"}
|
package/dist/asr.js
CHANGED
|
@@ -1,19 +1,132 @@
|
|
|
1
1
|
"use strict";
|
|
2
|
+
var __awaiter = (this && this.__awaiter) || function (thisArg, _arguments, P, generator) {
|
|
3
|
+
function adopt(value) { return value instanceof P ? value : new P(function (resolve) { resolve(value); }); }
|
|
4
|
+
return new (P || (P = Promise))(function (resolve, reject) {
|
|
5
|
+
function fulfilled(value) { try { step(generator.next(value)); } catch (e) { reject(e); } }
|
|
6
|
+
function rejected(value) { try { step(generator["throw"](value)); } catch (e) { reject(e); } }
|
|
7
|
+
function step(result) { result.done ? resolve(result.value) : adopt(result.value).then(fulfilled, rejected); }
|
|
8
|
+
step((generator = generator.apply(thisArg, _arguments || [])).next());
|
|
9
|
+
});
|
|
10
|
+
};
|
|
2
11
|
Object.defineProperty(exports, "__esModule", { value: true });
|
|
3
12
|
exports.asrMachine = void 0;
|
|
4
13
|
const xstate_1 = require("xstate");
|
|
5
14
|
const getToken_1 = require("./getToken");
|
|
6
15
|
const SpeechToText_1 = require("web-speech-cognitive-services/lib/SpeechServices/SpeechToText");
|
|
7
16
|
const REGION = "northeurope";
|
|
8
|
-
exports.asrMachine = (0, xstate_1.
|
|
17
|
+
exports.asrMachine = (0, xstate_1.setup)({
|
|
18
|
+
types: {
|
|
19
|
+
context: {},
|
|
20
|
+
events: {},
|
|
21
|
+
input: {},
|
|
22
|
+
},
|
|
23
|
+
actions: {
|
|
24
|
+
recStop: ({ context }) => {
|
|
25
|
+
context.wsaASRinstance.abort();
|
|
26
|
+
console.debug("[ASR] stopped");
|
|
27
|
+
},
|
|
28
|
+
},
|
|
29
|
+
actors: {
|
|
30
|
+
getToken: getToken_1.getToken,
|
|
31
|
+
ponyfill: (0, xstate_1.fromCallback)(({ sendBack, input }) => {
|
|
32
|
+
const { SpeechGrammarList, SpeechRecognition } = (0, SpeechToText_1.default)({
|
|
33
|
+
audioContext: input.audioContext,
|
|
34
|
+
speechRecognitionEndpointId: input
|
|
35
|
+
.speechRecognitionEndpointId,
|
|
36
|
+
credentials: {
|
|
37
|
+
region: REGION, // TODO
|
|
38
|
+
authorizationToken: input.azureAuthorizationToken,
|
|
39
|
+
},
|
|
40
|
+
});
|
|
41
|
+
sendBack({
|
|
42
|
+
type: "READY",
|
|
43
|
+
value: {
|
|
44
|
+
wsaASR: SpeechRecognition,
|
|
45
|
+
wsaGrammarList: SpeechGrammarList,
|
|
46
|
+
},
|
|
47
|
+
});
|
|
48
|
+
console.debug("[ASR] READY", input);
|
|
49
|
+
}),
|
|
50
|
+
recStart: (0, xstate_1.fromCallback)(({ sendBack, input }) => {
|
|
51
|
+
let asr = new input.wsaASR();
|
|
52
|
+
asr.grammars = new input.wsaGrammarList();
|
|
53
|
+
asr.grammars.phrases = input.phrases || [];
|
|
54
|
+
asr.lang = input.locale;
|
|
55
|
+
asr.continuous = true;
|
|
56
|
+
asr.interimResults = true;
|
|
57
|
+
asr.onresult = function (event) {
|
|
58
|
+
if (event.results[event.results.length - 1].isFinal) {
|
|
59
|
+
const transcript = event.results
|
|
60
|
+
.map((x) => x[0].transcript.replace(/\.$/, ""))
|
|
61
|
+
.join(" ");
|
|
62
|
+
const confidence = event.results
|
|
63
|
+
.map((x) => x[0].confidence)
|
|
64
|
+
.reduce((a, b) => a + b) / event.results.length;
|
|
65
|
+
const res = [
|
|
66
|
+
{
|
|
67
|
+
utterance: transcript,
|
|
68
|
+
confidence: confidence,
|
|
69
|
+
},
|
|
70
|
+
];
|
|
71
|
+
sendBack({
|
|
72
|
+
type: "RESULT",
|
|
73
|
+
value: res,
|
|
74
|
+
});
|
|
75
|
+
console.debug("[ASR] RESULT (pre-final)", res);
|
|
76
|
+
}
|
|
77
|
+
else {
|
|
78
|
+
sendBack({ type: "STARTSPEECH" });
|
|
79
|
+
}
|
|
80
|
+
};
|
|
81
|
+
asr.addEventListener("start", () => {
|
|
82
|
+
sendBack({ type: "STARTED", value: { wsaASRinstance: asr } });
|
|
83
|
+
});
|
|
84
|
+
// receive((event) => {
|
|
85
|
+
// console.debug("bla");
|
|
86
|
+
// if (event.type === "STOP") {
|
|
87
|
+
// asr.abort();
|
|
88
|
+
// }
|
|
89
|
+
// });
|
|
90
|
+
asr.start();
|
|
91
|
+
}),
|
|
92
|
+
nluPromise: (0, xstate_1.fromPromise)(({ input }) => __awaiter(void 0, void 0, void 0, function* () {
|
|
93
|
+
const response = yield fetch(new Request(input.endpoint, {
|
|
94
|
+
method: "POST",
|
|
95
|
+
headers: {
|
|
96
|
+
"Ocp-Apim-Subscription-Key": input.key,
|
|
97
|
+
"Content-Type": "application/json",
|
|
98
|
+
},
|
|
99
|
+
body: JSON.stringify({
|
|
100
|
+
kind: "Conversation",
|
|
101
|
+
analysisInput: {
|
|
102
|
+
conversationItem: {
|
|
103
|
+
id: "PARTICIPANT_ID_HERE",
|
|
104
|
+
text: input.query,
|
|
105
|
+
modality: "text",
|
|
106
|
+
language: "en-US", // TODO
|
|
107
|
+
participantId: "PARTICIPANT_ID_HERE",
|
|
108
|
+
},
|
|
109
|
+
},
|
|
110
|
+
parameters: {
|
|
111
|
+
projectName: input.projectName,
|
|
112
|
+
verbose: true,
|
|
113
|
+
deploymentName: input.deploymentName,
|
|
114
|
+
stringIndexType: "TextElement_V8",
|
|
115
|
+
},
|
|
116
|
+
}),
|
|
117
|
+
}));
|
|
118
|
+
return response.json();
|
|
119
|
+
})),
|
|
120
|
+
},
|
|
121
|
+
}).createMachine({
|
|
9
122
|
id: "asr",
|
|
10
|
-
types: {},
|
|
11
123
|
context: ({ input }) => ({
|
|
12
124
|
asrDefaultCompleteTimeout: input.asrDefaultCompleteTimeout || 0,
|
|
13
125
|
asrDefaultNoInputTimeout: input.asrDefaultNoInputTimeout || 5000,
|
|
14
126
|
locale: input.locale || "en-US",
|
|
15
127
|
audioContext: input.audioContext,
|
|
16
128
|
azureCredentials: input.azureCredentials,
|
|
129
|
+
azureLanguageCredentials: input.azureLanguageCredentials,
|
|
17
130
|
speechRecognitionEndpointId: input.speechRecognitionEndpointId,
|
|
18
131
|
}),
|
|
19
132
|
initial: "GetToken",
|
|
@@ -35,7 +148,7 @@ exports.asrMachine = (0, xstate_1.createMachine)({
|
|
|
35
148
|
on: {
|
|
36
149
|
START: {
|
|
37
150
|
target: "Recognising",
|
|
38
|
-
actions: (0, xstate_1.assign)({ params: ({ event }) => event.value }),
|
|
151
|
+
actions: (0, xstate_1.assign)({ params: ({ event }) => event.value || {} }),
|
|
39
152
|
},
|
|
40
153
|
},
|
|
41
154
|
},
|
|
@@ -62,15 +175,21 @@ exports.asrMachine = (0, xstate_1.createMachine)({
|
|
|
62
175
|
],
|
|
63
176
|
target: ".Match",
|
|
64
177
|
},
|
|
65
|
-
RECOGNISED:
|
|
66
|
-
|
|
67
|
-
|
|
68
|
-
|
|
69
|
-
|
|
70
|
-
|
|
71
|
-
|
|
72
|
-
|
|
73
|
-
|
|
178
|
+
RECOGNISED: [
|
|
179
|
+
{
|
|
180
|
+
target: ".NLURequest",
|
|
181
|
+
guard: ({ context }) => !!(context.params.nlu && context.azureLanguageCredentials),
|
|
182
|
+
},
|
|
183
|
+
{
|
|
184
|
+
target: "Ready",
|
|
185
|
+
actions: [
|
|
186
|
+
(0, xstate_1.sendParent)(({ context }) => ({
|
|
187
|
+
type: "RECOGNISED",
|
|
188
|
+
value: context.result,
|
|
189
|
+
})),
|
|
190
|
+
],
|
|
191
|
+
},
|
|
192
|
+
],
|
|
74
193
|
CONTROL: {
|
|
75
194
|
target: "Pause",
|
|
76
195
|
},
|
|
@@ -123,6 +242,58 @@ exports.asrMachine = (0, xstate_1.createMachine)({
|
|
|
123
242
|
}),
|
|
124
243
|
],
|
|
125
244
|
},
|
|
245
|
+
NLURequest: {
|
|
246
|
+
invoke: {
|
|
247
|
+
src: "nluPromise",
|
|
248
|
+
input: ({ context }) => {
|
|
249
|
+
let c;
|
|
250
|
+
typeof context.params.nlu === "boolean"
|
|
251
|
+
? (c = context.azureLanguageCredentials)
|
|
252
|
+
: (c = context.params.nlu);
|
|
253
|
+
return {
|
|
254
|
+
endpoint: c.endpoint,
|
|
255
|
+
key: c.key,
|
|
256
|
+
projectName: c.projectName,
|
|
257
|
+
deploymentName: c.deploymentName,
|
|
258
|
+
query: context.result[0].utterance,
|
|
259
|
+
};
|
|
260
|
+
},
|
|
261
|
+
onDone: [
|
|
262
|
+
{
|
|
263
|
+
actions: [
|
|
264
|
+
({ event }) => console.error("[ASR] no NLU prediction", event.output),
|
|
265
|
+
(0, xstate_1.sendParent)(({ context }) => ({
|
|
266
|
+
type: "RECOGNISED",
|
|
267
|
+
value: context.result,
|
|
268
|
+
})),
|
|
269
|
+
],
|
|
270
|
+
target: "#asr.Ready",
|
|
271
|
+
guard: ({ event }) => !(event.output.result || {}).prediction,
|
|
272
|
+
},
|
|
273
|
+
{
|
|
274
|
+
actions: [
|
|
275
|
+
({ event }) => console.debug("[ASR] NLU result", event.output.result.prediction),
|
|
276
|
+
(0, xstate_1.sendParent)(({ context, event }) => ({
|
|
277
|
+
type: "RECOGNISED",
|
|
278
|
+
value: context.result,
|
|
279
|
+
nluValue: event.output.result.prediction,
|
|
280
|
+
})),
|
|
281
|
+
],
|
|
282
|
+
target: "#asr.Ready",
|
|
283
|
+
},
|
|
284
|
+
],
|
|
285
|
+
onError: {
|
|
286
|
+
actions: [
|
|
287
|
+
({ event }) => console.error("[ASR]", event.error),
|
|
288
|
+
(0, xstate_1.sendParent)(({ context }) => ({
|
|
289
|
+
type: "RECOGNISED",
|
|
290
|
+
value: context.result,
|
|
291
|
+
})),
|
|
292
|
+
],
|
|
293
|
+
target: "#asr.Ready",
|
|
294
|
+
},
|
|
295
|
+
},
|
|
296
|
+
},
|
|
126
297
|
},
|
|
127
298
|
},
|
|
128
299
|
Pause: {
|
|
@@ -159,6 +330,7 @@ exports.asrMachine = (0, xstate_1.createMachine)({
|
|
|
159
330
|
],
|
|
160
331
|
},
|
|
161
332
|
onError: {
|
|
333
|
+
actions: ({ event }) => console.error("[ASR]", event.error),
|
|
162
334
|
target: "Fail",
|
|
163
335
|
},
|
|
164
336
|
},
|
|
@@ -176,76 +348,4 @@ exports.asrMachine = (0, xstate_1.createMachine)({
|
|
|
176
348
|
},
|
|
177
349
|
},
|
|
178
350
|
},
|
|
179
|
-
}, {
|
|
180
|
-
actions: {
|
|
181
|
-
recStop: ({ context }) => {
|
|
182
|
-
context.wsaASRinstance.abort();
|
|
183
|
-
console.debug("[ASR] stopped");
|
|
184
|
-
},
|
|
185
|
-
},
|
|
186
|
-
actors: {
|
|
187
|
-
getToken: getToken_1.getToken,
|
|
188
|
-
ponyfill: (0, xstate_1.fromCallback)(({ sendBack, input }) => {
|
|
189
|
-
const { SpeechGrammarList, SpeechRecognition } = (0, SpeechToText_1.default)({
|
|
190
|
-
audioContext: input.audioContext,
|
|
191
|
-
speechRecognitionEndpointId: input
|
|
192
|
-
.speechRecognitionEndpointId,
|
|
193
|
-
credentials: {
|
|
194
|
-
region: REGION, // TODO
|
|
195
|
-
authorizationToken: input.azureAuthorizationToken,
|
|
196
|
-
},
|
|
197
|
-
});
|
|
198
|
-
sendBack({
|
|
199
|
-
type: "READY",
|
|
200
|
-
value: {
|
|
201
|
-
wsaASR: SpeechRecognition,
|
|
202
|
-
wsaGrammarList: SpeechGrammarList,
|
|
203
|
-
},
|
|
204
|
-
});
|
|
205
|
-
console.debug("[ASR] READY", input);
|
|
206
|
-
}),
|
|
207
|
-
recStart: (0, xstate_1.fromCallback)(({ sendBack, input }) => {
|
|
208
|
-
let asr = new input.wsaASR();
|
|
209
|
-
asr.grammars = new input.wsaGrammarList();
|
|
210
|
-
asr.grammars.phrases = input.phrases || [];
|
|
211
|
-
asr.lang = input.locale;
|
|
212
|
-
asr.continuous = true;
|
|
213
|
-
asr.interimResults = true;
|
|
214
|
-
asr.onresult = function (event) {
|
|
215
|
-
if (event.results[event.results.length - 1].isFinal) {
|
|
216
|
-
const transcript = event.results
|
|
217
|
-
.map((x) => x[0].transcript.replace(/\.$/, ""))
|
|
218
|
-
.join(" ");
|
|
219
|
-
const confidence = event.results
|
|
220
|
-
.map((x) => x[0].confidence)
|
|
221
|
-
.reduce((a, b) => a + b) /
|
|
222
|
-
event.results.length;
|
|
223
|
-
const res = [
|
|
224
|
-
{
|
|
225
|
-
utterance: transcript,
|
|
226
|
-
confidence: confidence,
|
|
227
|
-
},
|
|
228
|
-
];
|
|
229
|
-
sendBack({
|
|
230
|
-
type: "RESULT",
|
|
231
|
-
value: res,
|
|
232
|
-
});
|
|
233
|
-
console.debug("[ASR] RESULT (pre-final)", res);
|
|
234
|
-
}
|
|
235
|
-
else {
|
|
236
|
-
sendBack({ type: "STARTSPEECH" });
|
|
237
|
-
}
|
|
238
|
-
};
|
|
239
|
-
asr.addEventListener("start", () => {
|
|
240
|
-
sendBack({ type: "STARTED", value: { wsaASRinstance: asr } });
|
|
241
|
-
});
|
|
242
|
-
// receive((event) => {
|
|
243
|
-
// console.debug("bla");
|
|
244
|
-
// if (event.type === "STOP") {
|
|
245
|
-
// asr.abort();
|
|
246
|
-
// }
|
|
247
|
-
// });
|
|
248
|
-
asr.start();
|
|
249
|
-
}),
|
|
250
|
-
},
|
|
251
351
|
});
|
package/dist/speechstate.d.ts
CHANGED
|
@@ -1,11 +1,12 @@
|
|
|
1
|
-
import { Settings, Agenda, Hypothesis } from "./types";
|
|
1
|
+
import { Settings, Agenda, Hypothesis, RecogniseParameters } from "./types";
|
|
2
2
|
interface SSContext {
|
|
3
3
|
settings: Settings;
|
|
4
4
|
audioContext?: AudioContext;
|
|
5
5
|
asrRef?: any;
|
|
6
6
|
ttsRef?: any;
|
|
7
7
|
}
|
|
8
|
-
|
|
8
|
+
/** events sent to the spawned `speechstate` machine **/
|
|
9
|
+
type SSEventExtIn = {
|
|
9
10
|
type: "PREPARE";
|
|
10
11
|
} | {
|
|
11
12
|
type: "CONTROL";
|
|
@@ -14,26 +15,34 @@ type SSEvent = {
|
|
|
14
15
|
} | {
|
|
15
16
|
type: "SPEAK";
|
|
16
17
|
value: Agenda;
|
|
17
|
-
} | {
|
|
18
|
-
type: "TTS_READY";
|
|
19
|
-
} | {
|
|
20
|
-
type: "TTS_STARTED";
|
|
21
|
-
} | {
|
|
22
|
-
type: "TTS_ERROR";
|
|
23
|
-
} | {
|
|
24
|
-
type: "SPEAK_COMPLETE";
|
|
25
|
-
} | {
|
|
26
|
-
type: "ASR_READY";
|
|
27
18
|
} | {
|
|
28
19
|
type: "LISTEN";
|
|
20
|
+
value: RecogniseParameters;
|
|
21
|
+
};
|
|
22
|
+
/** for sendParent, not type-checked */
|
|
23
|
+
type SSEventExtOut = {
|
|
24
|
+
type: "ASR_NOINPUT";
|
|
25
|
+
} | {
|
|
26
|
+
type: "ASRTTS_READY";
|
|
29
27
|
} | {
|
|
30
28
|
type: "ASR_STARTED";
|
|
31
29
|
} | {
|
|
32
|
-
type: "
|
|
30
|
+
type: "TTS_STARTED";
|
|
31
|
+
} | {
|
|
32
|
+
type: "SPEAK_COMPLETE";
|
|
33
33
|
} | {
|
|
34
34
|
type: "RECOGNISED";
|
|
35
35
|
value: Hypothesis[];
|
|
36
|
+
nluValue?: any;
|
|
37
|
+
};
|
|
38
|
+
type SSEventIntIn = {
|
|
39
|
+
type: "TTS_READY";
|
|
40
|
+
} | {
|
|
41
|
+
type: "ASR_READY";
|
|
42
|
+
} | {
|
|
43
|
+
type: "TTS_ERROR";
|
|
36
44
|
};
|
|
45
|
+
type SSEvent = SSEventIntIn | SSEventExtIn | SSEventExtOut;
|
|
37
46
|
declare const speechstate: import("xstate").StateMachine<SSContext, SSEvent, Record<string, import("xstate").AnyActorRef>, import("xstate").ProvidedActor, import("xstate").ParameterizedObject, import("xstate").ParameterizedObject, string, import("xstate").StateValue, string, Settings, {}, import("xstate").ResolveTypegenMeta<import("xstate").TypegenDisabled, SSEvent, import("xstate").ProvidedActor, import("xstate").ParameterizedObject, import("xstate").ParameterizedObject, string, string>>;
|
|
38
47
|
export { speechstate };
|
|
39
48
|
//# sourceMappingURL=speechstate.d.ts.map
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"speechstate.d.ts","sourceRoot":"","sources":["../src/speechstate.ts"],"names":[],"mappings":"AAIA,OAAO,EAAE,QAAQ,EAAE,MAAM,EAAE,UAAU,EAAE,MAAM,SAAS,CAAC;
|
|
1
|
+
{"version":3,"file":"speechstate.d.ts","sourceRoot":"","sources":["../src/speechstate.ts"],"names":[],"mappings":"AAIA,OAAO,EAAE,QAAQ,EAAE,MAAM,EAAE,UAAU,EAAE,mBAAmB,EAAE,MAAM,SAAS,CAAC;AAC5E,UAAU,SAAS;IACjB,QAAQ,EAAE,QAAQ,CAAC;IACnB,YAAY,CAAC,EAAE,YAAY,CAAC;IAC5B,MAAM,CAAC,EAAE,GAAG,CAAC;IACb,MAAM,CAAC,EAAE,GAAG,CAAC;CACd;AAED,wDAAwD;AACxD,KAAK,YAAY,GACb;IAAE,IAAI,EAAE,SAAS,CAAA;CAAE,GACnB;IAAE,IAAI,EAAE,SAAS,CAAA;CAAE,GACnB;IAAE,IAAI,EAAE,MAAM,CAAA;CAAE,GAChB;IAAE,IAAI,EAAE,OAAO,CAAC;IAAC,KAAK,EAAE,MAAM,CAAA;CAAE,GAChC;IAAE,IAAI,EAAE,QAAQ,CAAC;IAAC,KAAK,EAAE,mBAAmB,CAAA;CAAE,CAAC;AAEnD,uCAAuC;AACvC,KAAK,aAAa,GACd;IAAE,IAAI,EAAE,aAAa,CAAA;CAAE,GACvB;IAAE,IAAI,EAAE,cAAc,CAAA;CAAE,GACxB;IAAE,IAAI,EAAE,aAAa,CAAA;CAAE,GACvB;IAAE,IAAI,EAAE,aAAa,CAAA;CAAE,GACvB;IAAE,IAAI,EAAE,gBAAgB,CAAA;CAAE,GAC1B;IAAE,IAAI,EAAE,YAAY,CAAC;IAAC,KAAK,EAAE,UAAU,EAAE,CAAC;IAAC,QAAQ,CAAC,EAAE,GAAG,CAAA;CAAE,CAAC;AAEhE,KAAK,YAAY,GACb;IAAE,IAAI,EAAE,WAAW,CAAA;CAAE,GACrB;IAAE,IAAI,EAAE,WAAW,CAAA;CAAE,GACrB;IAAE,IAAI,EAAE,WAAW,CAAA;CAAE,CAAC;AAE1B,KAAK,OAAO,GAAG,YAAY,GAAG,YAAY,GAAG,aAAa,CAAC;AAE3D,QAAA,MAAM,WAAW,odA6PhB,CAAC;AAEF,OAAO,EAAE,WAAW,EAAE,CAAC"}
|
package/dist/speechstate.js
CHANGED
|
@@ -48,6 +48,7 @@ const speechstate = (0, xstate_1.createMachine)({
|
|
|
48
48
|
locale: context.settings.locale,
|
|
49
49
|
audioContext: context.audioContext,
|
|
50
50
|
azureCredentials: context.settings.azureCredentials,
|
|
51
|
+
azureLanguageCredentials: context.settings.azureLanguageCredentials,
|
|
51
52
|
speechRecognitionEndpointId: context.settings.speechRecognitionEndpointId,
|
|
52
53
|
},
|
|
53
54
|
});
|
|
@@ -187,10 +188,11 @@ const speechstate = (0, xstate_1.createMachine)({
|
|
|
187
188
|
},
|
|
188
189
|
RECOGNISED: {
|
|
189
190
|
actions: [
|
|
190
|
-
({ event }) => console.debug("[ASR→SpSt] RECOGNISED", event.value),
|
|
191
|
+
({ event }) => console.debug("[ASR→SpSt] RECOGNISED", event.value, event.nluValue),
|
|
191
192
|
(0, xstate_1.sendParent)(({ event }) => ({
|
|
192
193
|
type: "RECOGNISED",
|
|
193
194
|
value: event.value,
|
|
195
|
+
nluValue: event.nluValue,
|
|
194
196
|
})),
|
|
195
197
|
],
|
|
196
198
|
target: "Idle",
|
package/dist/types.d.ts
CHANGED
|
@@ -1,10 +1,22 @@
|
|
|
1
|
-
export interface
|
|
1
|
+
export interface AzureSpeechCredentials {
|
|
2
2
|
endpoint: string;
|
|
3
3
|
key: string;
|
|
4
4
|
}
|
|
5
|
+
/**
|
|
6
|
+
* @deprecated use `AzureSpeechCredentials` instead
|
|
7
|
+
*/
|
|
8
|
+
export interface AzureCredentials extends AzureSpeechCredentials {
|
|
9
|
+
}
|
|
10
|
+
export interface AzureLanguageCredentials {
|
|
11
|
+
endpoint: string;
|
|
12
|
+
key: string;
|
|
13
|
+
projectName: string;
|
|
14
|
+
deploymentName: string;
|
|
15
|
+
}
|
|
5
16
|
export interface Settings {
|
|
6
17
|
locale?: string;
|
|
7
|
-
azureCredentials: string | AzureCredentials;
|
|
18
|
+
azureCredentials: string | AzureCredentials | AzureSpeechCredentials;
|
|
19
|
+
azureLanguageCredentials?: AzureLanguageCredentials;
|
|
8
20
|
asrDefaultCompleteTimeout?: number;
|
|
9
21
|
asrDefaultNoInputTimeout?: number;
|
|
10
22
|
ttsDefaultVoice?: string;
|
|
@@ -24,5 +36,6 @@ export interface RecogniseParameters {
|
|
|
24
36
|
completeTimeout?: number;
|
|
25
37
|
locale?: string;
|
|
26
38
|
hints?: string[];
|
|
39
|
+
nlu?: boolean | AzureLanguageCredentials;
|
|
27
40
|
}
|
|
28
41
|
//# sourceMappingURL=types.d.ts.map
|
package/dist/types.d.ts.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"types.d.ts","sourceRoot":"","sources":["../src/types.ts"],"names":[],"mappings":"AAAA,MAAM,WAAW,
|
|
1
|
+
{"version":3,"file":"types.d.ts","sourceRoot":"","sources":["../src/types.ts"],"names":[],"mappings":"AAAA,MAAM,WAAW,sBAAsB;IACrC,QAAQ,EAAE,MAAM,CAAC;IACjB,GAAG,EAAE,MAAM,CAAC;CACb;AAED;;GAEG;AACH,MAAM,WAAW,gBAAiB,SAAQ,sBAAsB;CAAG;AAEnE,MAAM,WAAW,wBAAwB;IACvC,QAAQ,EAAE,MAAM,CAAC;IACjB,GAAG,EAAE,MAAM,CAAC;IACZ,WAAW,EAAE,MAAM,CAAC;IACpB,cAAc,EAAE,MAAM,CAAC;CACxB;AAED,MAAM,WAAW,QAAQ;IACvB,MAAM,CAAC,EAAE,MAAM,CAAC;IAChB,gBAAgB,EAAE,MAAM,GAAG,gBAAgB,GAAG,sBAAsB,CAAC;IACrE,wBAAwB,CAAC,EAAE,wBAAwB,CAAC;IACpD,yBAAyB,CAAC,EAAE,MAAM,CAAC;IACnC,wBAAwB,CAAC,EAAE,MAAM,CAAC;IAClC,eAAe,CAAC,EAAE,MAAM,CAAC;IACzB,2BAA2B,CAAC,EAAE,MAAM,CAAC;CACtC;AAED,MAAM,WAAW,MAAM;IACrB,SAAS,EAAE,MAAM,CAAC;IAClB,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,MAAM,CAAC,EAAE,cAAc,CAAC,MAAM,CAAC,CAAC;CACjC;AAED,MAAM,WAAW,UAAU;IACzB,SAAS,EAAE,MAAM,CAAC;IAClB,UAAU,EAAE,MAAM,CAAC;CACpB;AAED,MAAM,WAAW,mBAAmB;IAClC,cAAc,CAAC,EAAE,MAAM,CAAC;IACxB,eAAe,CAAC,EAAE,MAAM,CAAC;IACzB,MAAM,CAAC,EAAE,MAAM,CAAC;IAChB,KAAK,CAAC,EAAE,MAAM,EAAE,CAAC;IACjB,GAAG,CAAC,EAAE,OAAO,GAAG,wBAAwB,CAAC;CAC1C"}
|
package/package.json
CHANGED
|
@@ -1,18 +1,20 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "speechstate",
|
|
3
|
-
"version": "2.0.0-beta.
|
|
3
|
+
"version": "2.0.0-beta.7",
|
|
4
4
|
"license": "GPL-3.0",
|
|
5
5
|
"homepage": "http://localhost/speechstate",
|
|
6
6
|
"main": "./dist/index.js",
|
|
7
7
|
"types": "./dist/index.d.ts",
|
|
8
8
|
"files": ["/dist"],
|
|
9
9
|
"dependencies": {
|
|
10
|
+
"@statelyai/inspect": "^0.2.3",
|
|
10
11
|
"microsoft-cognitiveservices-speech-sdk": "^1.31.0",
|
|
11
12
|
"web-speech-cognitive-services": "^7.1.3",
|
|
12
13
|
"xstate": "^5.5.1"
|
|
13
14
|
},
|
|
14
15
|
"scripts": {
|
|
15
16
|
"dev": "vite",
|
|
17
|
+
"compile": "tsc",
|
|
16
18
|
"test": "jest --silent=false"
|
|
17
19
|
},
|
|
18
20
|
"eslintConfig": {
|