rasa-pro 3.11.0a4.dev3__py3-none-any.whl → 3.11.0rc1__py3-none-any.whl

rasa/core/channels/inspector/dist/assets/{xychartDiagram-2b33534f-f92cfea9.js → xychartDiagram-2b33534f-fd3db0d5.js}

@@ -1,4 +1,4 @@
-import{Z as zt,$ as ot,X as wt,W as Ft,s as Nt,g as Xt,B as Yt,D as St,a as Ht,b as $t,E as Ut,l as Ct,U as qt,i as jt,d as Gt}from"./index-efdd30c1.js";import{c as Qt}from"./createText-62fc7601-e5ee049d.js";import{i as Kt}from"./init-77b53fdd.js";import{o as Zt}from"./ordinal-ba9b4969.js";import{l as ft}from"./linear-08814e93.js";import{l as pt}from"./line-40186f1f.js";import"./array-9f3ba611.js";import"./path-53f90ab3.js";function Jt(e,t,i){e=+e,t=+t,i=(n=arguments.length)<2?(t=e,e=0,1):n<3?1:+i;for(var s=-1,n=Math.max(0,Math.ceil((t-e)/i))|0,o=new Array(n);++s<n;)o[s]=e+s*i;return o}function st(){var e=Zt().unknown(void 0),t=e.domain,i=e.range,s=0,n=1,o,c,f=!1,d=0,R=0,_=.5;delete e.unknown;function A(){var m=t().length,T=n<s,S=T?n:s,P=T?s:n;o=(P-S)/Math.max(1,m-d+R*2),f&&(o=Math.floor(o)),S+=(P-S-o*(m-d))*_,c=o*(1-d),f&&(S=Math.round(S),c=Math.round(c));var p=Jt(m).map(function(C){return S+o*C});return i(T?p.reverse():p)}return e.domain=function(m){return arguments.length?(t(m),A()):t()},e.range=function(m){return arguments.length?([s,n]=m,s=+s,n=+n,A()):[s,n]},e.rangeRound=function(m){return[s,n]=m,s=+s,n=+n,f=!0,A()},e.bandwidth=function(){return c},e.step=function(){return o},e.round=function(m){return arguments.length?(f=!!m,A()):f},e.padding=function(m){return arguments.length?(d=Math.min(1,R=+m),A()):d},e.paddingInner=function(m){return arguments.length?(d=Math.min(1,m),A()):d},e.paddingOuter=function(m){return arguments.length?(R=+m,A()):R},e.align=function(m){return arguments.length?(_=Math.max(0,Math.min(1,m)),A()):_},e.copy=function(){return st(t(),[s,n]).round(f).paddingInner(d).paddingOuter(R).align(_)},Kt.apply(A(),arguments)}var nt=function(){var e=function(V,r,l,u){for(l=l||{},u=V.length;u--;l[V[u]]=r);return l},t=[1,10,12,14,16,18,19,21,23],i=[2,6],s=[1,3],n=[1,5],o=[1,6],c=[1,7],f=[1,5,10,12,14,16,18,19,21,23,34,35,36],d=[1,25],R=[1,26],_=[1,28],A=[1,29],m=[1,30],T=[1,31],S=[1,32],P=[1,33],p=[1,34],C=[1,35],h=[1,36],L=[1,37],z=[1,43],lt=[1,42],ct=[1,47],$=[1,50],w=[1,10,12,14,16,18,19,21,23,34,35,36],Q=[1,10,12,14,16,18,19,21,23,24,26,27,28,34,35,36],E=[1,10,12,14,16,18,19,21,23,24,26,27,28,34,35,36,41,42,43,44,45,46,47,48,49,50],ut=[1,64],K={trace:function(){},yy:{},symbols_:{error:2,start:3,eol:4,XYCHART:5,chartConfig:6,document:7,CHART_ORIENTATION:8,statement:9,title:10,text:11,X_AXIS:12,parseXAxis:13,Y_AXIS:14,parseYAxis:15,LINE:16,plotData:17,BAR:18,acc_title:19,acc_title_value:20,acc_descr:21,acc_descr_value:22,acc_descr_multiline_value:23,SQUARE_BRACES_START:24,commaSeparatedNumbers:25,SQUARE_BRACES_END:26,NUMBER_WITH_DECIMAL:27,COMMA:28,xAxisData:29,bandData:30,ARROW_DELIMITER:31,commaSeparatedTexts:32,yAxisData:33,NEWLINE:34,SEMI:35,EOF:36,alphaNum:37,STR:38,MD_STR:39,alphaNumToken:40,AMP:41,NUM:42,ALPHA:43,PLUS:44,EQUALS:45,MULT:46,DOT:47,BRKT:48,MINUS:49,UNDERSCORE:50,$accept:0,$end:1},terminals_:{2:"error",5:"XYCHART",8:"CHART_ORIENTATION",10:"title",12:"X_AXIS",14:"Y_AXIS",16:"LINE",18:"BAR",19:"acc_title",20:"acc_title_value",21:"acc_descr",22:"acc_descr_value",23:"acc_descr_multiline_value",24:"SQUARE_BRACES_START",26:"SQUARE_BRACES_END",27:"NUMBER_WITH_DECIMAL",28:"COMMA",31:"ARROW_DELIMITER",34:"NEWLINE",35:"SEMI",36:"EOF",38:"STR",39:"MD_STR",41:"AMP",42:"NUM",43:"ALPHA",44:"PLUS",45:"EQUALS",46:"MULT",47:"DOT",48:"BRKT",49:"MINUS",50:"UNDERSCORE"},productions_:[0,[3,2],[3,3],[3,2],[3,1],[6,1],[7,0],[7,2],[9,2],[9,2],[9,2],[9,2],[9,2],[9,3],[9,2],[9,3],[9,2],[9,2],[9,1],[17,3],[25,3],[25,1],[13,1],[13,2],[13,1],[29,1],[29,3],[30,3],[32,3],[32,1],[15,1],[15,2],[15,1],[33,3],[4,1],[4,1],[4,1],[11,1],[11,1],[11,1],[37,1],[37,2],[40,1],[40,1],[40,1],[40,1],[40,1],[40,1],[40,1],[40,1],[40,1],[40,1]],performAction:function(r,l,u,g,b,a,F){var x=a.length-1;switch(b){case 5:g.setOrientation(a[x]);break;case 9:g.setDiagramTitle(a[x].text.trim());break;case 12:g.setLineData({text:"",type:"text"},a[x]);break;case 13:g.setLineData(a[x-1],a[x]);break;case 14:g.setBarData({text:"",type:"text"},a[x]);break;case 15:g.setBarData(a[x-1],a[x]);break;case 16:this.$=a[x].trim(),g.setAccTitle(this.$);break;case 17:case 18:this.$=a[x].trim(),g.setAccDescription(this.$);break;case 19:this.$=a[x-1];break;case 20:this.$=[Number(a[x-2]),...a[x]];break;case 21:this.$=[Number(a[x])];break;case 22:g.setXAxisTitle(a[x]);break;case 23:g.setXAxisTitle(a[x-1]);break;case 24:g.setXAxisTitle({type:"text",text:""});break;case 25:g.setXAxisBand(a[x]);break;case 26:g.setXAxisRangeData(Number(a[x-2]),Number(a[x]));break;case 27:this.$=a[x-1];break;case 28:this.$=[a[x-2],...a[x]];break;case 29:this.$=[a[x]];break;case 30:g.setYAxisTitle(a[x]);break;case 31:g.setYAxisTitle(a[x-1]);break;case 32:g.setYAxisTitle({type:"text",text:""});break;case 33:g.setYAxisRangeData(Number(a[x-2]),Number(a[x]));break;case 37:this.$={text:a[x],type:"text"};break;case 38:this.$={text:a[x],type:"text"};break;case 39:this.$={text:a[x],type:"markdown"};break;case 40:this.$=a[x];break;case 41:this.$=a[x-1]+""+a[x];break}},table:[e(t,i,{3:1,4:2,7:4,5:s,34:n,35:o,36:c}),{1:[3]},e(t,i,{4:2,7:4,3:8,5:s,34:n,35:o,36:c}),e(t,i,{4:2,7:4,6:9,3:10,5:s,8:[1,11],34:n,35:o,36:c}),{1:[2,4],9:12,10:[1,13],12:[1,14],14:[1,15],16:[1,16],18:[1,17],19:[1,18],21:[1,19],23:[1,20]},e(f,[2,34]),e(f,[2,35]),e(f,[2,36]),{1:[2,1]},e(t,i,{4:2,7:4,3:21,5:s,34:n,35:o,36:c}),{1:[2,3]},e(f,[2,5]),e(t,[2,7],{4:22,34:n,35:o,36:c}),{11:23,37:24,38:d,39:R,40:27,41:_,42:A,43:m,44:T,45:S,46:P,47:p,48:C,49:h,50:L},{11:39,13:38,24:z,27:lt,29:40,30:41,37:24,38:d,39:R,40:27,41:_,42:A,43:m,44:T,45:S,46:P,47:p,48:C,49:h,50:L},{11:45,15:44,27:ct,33:46,37:24,38:d,39:R,40:27,41:_,42:A,43:m,44:T,45:S,46:P,47:p,48:C,49:h,50:L},{11:49,17:48,24:$,37:24,38:d,39:R,40:27,41:_,42:A,43:m,44:T,45:S,46:P,47:p,48:C,49:h,50:L},{11:52,17:51,24:$,37:24,38:d,39:R,40:27,41:_,42:A,43:m,44:T,45:S,46:P,47:p,48:C,49:h,50:L},{20:[1,53]},{22:[1,54]},e(w,[2,18]),{1:[2,2]},e(w,[2,8]),e(w,[2,9]),e(Q,[2,37],{40:55,41:_,42:A,43:m,44:T,45:S,46:P,47:p,48:C,49:h,50:L}),e(Q,[2,38]),e(Q,[2,39]),e(E,[2,40]),e(E,[2,42]),e(E,[2,43]),e(E,[2,44]),e(E,[2,45]),e(E,[2,46]),e(E,[2,47]),e(E,[2,48]),e(E,[2,49]),e(E,[2,50]),e(E,[2,51]),e(w,[2,10]),e(w,[2,22],{30:41,29:56,24:z,27:lt}),e(w,[2,24]),e(w,[2,25]),{31:[1,57]},{11:59,32:58,37:24,38:d,39:R,40:27,41:_,42:A,43:m,44:T,45:S,46:P,47:p,48:C,49:h,50:L},e(w,[2,11]),e(w,[2,30],{33:60,27:ct}),e(w,[2,32]),{31:[1,61]},e(w,[2,12]),{17:62,24:$},{25:63,27:ut},e(w,[2,14]),{17:65,24:$},e(w,[2,16]),e(w,[2,17]),e(E,[2,41]),e(w,[2,23]),{27:[1,66]},{26:[1,67]},{26:[2,29],28:[1,68]},e(w,[2,31]),{27:[1,69]},e(w,[2,13]),{26:[1,70]},{26:[2,21],28:[1,71]},e(w,[2,15]),e(w,[2,26]),e(w,[2,27]),{11:59,32:72,37:24,38:d,39:R,40:27,41:_,42:A,43:m,44:T,45:S,46:P,47:p,48:C,49:h,50:L},e(w,[2,33]),e(w,[2,19]),{25:73,27:ut},{26:[2,28]},{26:[2,20]}],defaultActions:{8:[2,1],10:[2,3],21:[2,2],72:[2,28],73:[2,20]},parseError:function(r,l){if(l.recoverable)this.trace(r);else{var u=new Error(r);throw u.hash=l,u}},parse:function(r){var l=this,u=[0],g=[],b=[null],a=[],F=this.table,x="",U=0,gt=0,Vt=2,xt=1,Bt=a.slice.call(arguments,1),k=Object.create(this.lexer),B={yy:{}};for(var J in this.yy)Object.prototype.hasOwnProperty.call(this.yy,J)&&(B.yy[J]=this.yy[J]);k.setInput(r,B.yy),B.yy.lexer=k,B.yy.parser=this,typeof k.yylloc>"u"&&(k.yylloc={});var tt=k.yylloc;a.push(tt);var Wt=k.options&&k.options.ranges;typeof B.yy.parseError=="function"?this.parseError=B.yy.parseError:this.parseError=Object.getPrototypeOf(this).parseError;function Ot(){var I;return I=g.pop()||k.lex()||xt,typeof I!="number"&&(I instanceof Array&&(g=I,I=g.pop()),I=l.symbols_[I]||I),I}for(var D,W,v,it,O={},q,M,dt,j;;){if(W=u[u.length-1],this.defaultActions[W]?v=this.defaultActions[W]:((D===null||typeof D>"u")&&(D=Ot()),v=F[W]&&F[W][D]),typeof v>"u"||!v.length||!v[0]){var et="";j=[];for(q in F[W])this.terminals_[q]&&q>Vt&&j.push("'"+this.terminals_[q]+"'");k.showPosition?et="Parse error on line "+(U+1)+`:
+import{Z as zt,$ as ot,X as wt,W as Ft,s as Nt,g as Xt,B as Yt,D as St,a as Ht,b as $t,E as Ut,l as Ct,U as qt,i as jt,d as Gt}from"./index-e7cef9de.js";import{c as Qt}from"./createText-62fc7601-b0ec81d6.js";import{i as Kt}from"./init-77b53fdd.js";import{o as Zt}from"./ordinal-ba9b4969.js";import{l as ft}from"./linear-4776f780.js";import{l as pt}from"./line-7c59abb6.js";import"./array-9f3ba611.js";import"./path-53f90ab3.js";function Jt(e,t,i){e=+e,t=+t,i=(n=arguments.length)<2?(t=e,e=0,1):n<3?1:+i;for(var s=-1,n=Math.max(0,Math.ceil((t-e)/i))|0,o=new Array(n);++s<n;)o[s]=e+s*i;return o}function st(){var e=Zt().unknown(void 0),t=e.domain,i=e.range,s=0,n=1,o,c,f=!1,d=0,R=0,_=.5;delete e.unknown;function A(){var m=t().length,T=n<s,S=T?n:s,P=T?s:n;o=(P-S)/Math.max(1,m-d+R*2),f&&(o=Math.floor(o)),S+=(P-S-o*(m-d))*_,c=o*(1-d),f&&(S=Math.round(S),c=Math.round(c));var p=Jt(m).map(function(C){return S+o*C});return i(T?p.reverse():p)}return e.domain=function(m){return arguments.length?(t(m),A()):t()},e.range=function(m){return arguments.length?([s,n]=m,s=+s,n=+n,A()):[s,n]},e.rangeRound=function(m){return[s,n]=m,s=+s,n=+n,f=!0,A()},e.bandwidth=function(){return c},e.step=function(){return o},e.round=function(m){return arguments.length?(f=!!m,A()):f},e.padding=function(m){return arguments.length?(d=Math.min(1,R=+m),A()):d},e.paddingInner=function(m){return arguments.length?(d=Math.min(1,m),A()):d},e.paddingOuter=function(m){return arguments.length?(R=+m,A()):R},e.align=function(m){return arguments.length?(_=Math.max(0,Math.min(1,m)),A()):_},e.copy=function(){return st(t(),[s,n]).round(f).paddingInner(d).paddingOuter(R).align(_)},Kt.apply(A(),arguments)}var nt=function(){var e=function(V,r,l,u){for(l=l||{},u=V.length;u--;l[V[u]]=r);return l},t=[1,10,12,14,16,18,19,21,23],i=[2,6],s=[1,3],n=[1,5],o=[1,6],c=[1,7],f=[1,5,10,12,14,16,18,19,21,23,34,35,36],d=[1,25],R=[1,26],_=[1,28],A=[1,29],m=[1,30],T=[1,31],S=[1,32],P=[1,33],p=[1,34],C=[1,35],h=[1,36],L=[1,37],z=[1,43],lt=[1,42],ct=[1,47],$=[1,50],w=[1,10,12,14,16,18,19,21,23,34,35,36],Q=[1,10,12,14,16,18,19,21,23,24,26,27,28,34,35,36],E=[1,10,12,14,16,18,19,21,23,24,26,27,28,34,35,36,41,42,43,44,45,46,47,48,49,50],ut=[1,64],K={trace:function(){},yy:{},symbols_:{error:2,start:3,eol:4,XYCHART:5,chartConfig:6,document:7,CHART_ORIENTATION:8,statement:9,title:10,text:11,X_AXIS:12,parseXAxis:13,Y_AXIS:14,parseYAxis:15,LINE:16,plotData:17,BAR:18,acc_title:19,acc_title_value:20,acc_descr:21,acc_descr_value:22,acc_descr_multiline_value:23,SQUARE_BRACES_START:24,commaSeparatedNumbers:25,SQUARE_BRACES_END:26,NUMBER_WITH_DECIMAL:27,COMMA:28,xAxisData:29,bandData:30,ARROW_DELIMITER:31,commaSeparatedTexts:32,yAxisData:33,NEWLINE:34,SEMI:35,EOF:36,alphaNum:37,STR:38,MD_STR:39,alphaNumToken:40,AMP:41,NUM:42,ALPHA:43,PLUS:44,EQUALS:45,MULT:46,DOT:47,BRKT:48,MINUS:49,UNDERSCORE:50,$accept:0,$end:1},terminals_:{2:"error",5:"XYCHART",8:"CHART_ORIENTATION",10:"title",12:"X_AXIS",14:"Y_AXIS",16:"LINE",18:"BAR",19:"acc_title",20:"acc_title_value",21:"acc_descr",22:"acc_descr_value",23:"acc_descr_multiline_value",24:"SQUARE_BRACES_START",26:"SQUARE_BRACES_END",27:"NUMBER_WITH_DECIMAL",28:"COMMA",31:"ARROW_DELIMITER",34:"NEWLINE",35:"SEMI",36:"EOF",38:"STR",39:"MD_STR",41:"AMP",42:"NUM",43:"ALPHA",44:"PLUS",45:"EQUALS",46:"MULT",47:"DOT",48:"BRKT",49:"MINUS",50:"UNDERSCORE"},productions_:[0,[3,2],[3,3],[3,2],[3,1],[6,1],[7,0],[7,2],[9,2],[9,2],[9,2],[9,2],[9,2],[9,3],[9,2],[9,3],[9,2],[9,2],[9,1],[17,3],[25,3],[25,1],[13,1],[13,2],[13,1],[29,1],[29,3],[30,3],[32,3],[32,1],[15,1],[15,2],[15,1],[33,3],[4,1],[4,1],[4,1],[11,1],[11,1],[11,1],[37,1],[37,2],[40,1],[40,1],[40,1],[40,1],[40,1],[40,1],[40,1],[40,1],[40,1],[40,1]],performAction:function(r,l,u,g,b,a,F){var x=a.length-1;switch(b){case 5:g.setOrientation(a[x]);break;case 9:g.setDiagramTitle(a[x].text.trim());break;case 12:g.setLineData({text:"",type:"text"},a[x]);break;case 13:g.setLineData(a[x-1],a[x]);break;case 14:g.setBarData({text:"",type:"text"},a[x]);break;case 15:g.setBarData(a[x-1],a[x]);break;case 16:this.$=a[x].trim(),g.setAccTitle(this.$);break;case 17:case 18:this.$=a[x].trim(),g.setAccDescription(this.$);break;case 19:this.$=a[x-1];break;case 20:this.$=[Number(a[x-2]),...a[x]];break;case 21:this.$=[Number(a[x])];break;case 22:g.setXAxisTitle(a[x]);break;case 23:g.setXAxisTitle(a[x-1]);break;case 24:g.setXAxisTitle({type:"text",text:""});break;case 25:g.setXAxisBand(a[x]);break;case 26:g.setXAxisRangeData(Number(a[x-2]),Number(a[x]));break;case 27:this.$=a[x-1];break;case 28:this.$=[a[x-2],...a[x]];break;case 29:this.$=[a[x]];break;case 30:g.setYAxisTitle(a[x]);break;case 31:g.setYAxisTitle(a[x-1]);break;case 32:g.setYAxisTitle({type:"text",text:""});break;case 33:g.setYAxisRangeData(Number(a[x-2]),Number(a[x]));break;case 37:this.$={text:a[x],type:"text"};break;case 38:this.$={text:a[x],type:"text"};break;case 39:this.$={text:a[x],type:"markdown"};break;case 40:this.$=a[x];break;case 41:this.$=a[x-1]+""+a[x];break}},table:[e(t,i,{3:1,4:2,7:4,5:s,34:n,35:o,36:c}),{1:[3]},e(t,i,{4:2,7:4,3:8,5:s,34:n,35:o,36:c}),e(t,i,{4:2,7:4,6:9,3:10,5:s,8:[1,11],34:n,35:o,36:c}),{1:[2,4],9:12,10:[1,13],12:[1,14],14:[1,15],16:[1,16],18:[1,17],19:[1,18],21:[1,19],23:[1,20]},e(f,[2,34]),e(f,[2,35]),e(f,[2,36]),{1:[2,1]},e(t,i,{4:2,7:4,3:21,5:s,34:n,35:o,36:c}),{1:[2,3]},e(f,[2,5]),e(t,[2,7],{4:22,34:n,35:o,36:c}),{11:23,37:24,38:d,39:R,40:27,41:_,42:A,43:m,44:T,45:S,46:P,47:p,48:C,49:h,50:L},{11:39,13:38,24:z,27:lt,29:40,30:41,37:24,38:d,39:R,40:27,41:_,42:A,43:m,44:T,45:S,46:P,47:p,48:C,49:h,50:L},{11:45,15:44,27:ct,33:46,37:24,38:d,39:R,40:27,41:_,42:A,43:m,44:T,45:S,46:P,47:p,48:C,49:h,50:L},{11:49,17:48,24:$,37:24,38:d,39:R,40:27,41:_,42:A,43:m,44:T,45:S,46:P,47:p,48:C,49:h,50:L},{11:52,17:51,24:$,37:24,38:d,39:R,40:27,41:_,42:A,43:m,44:T,45:S,46:P,47:p,48:C,49:h,50:L},{20:[1,53]},{22:[1,54]},e(w,[2,18]),{1:[2,2]},e(w,[2,8]),e(w,[2,9]),e(Q,[2,37],{40:55,41:_,42:A,43:m,44:T,45:S,46:P,47:p,48:C,49:h,50:L}),e(Q,[2,38]),e(Q,[2,39]),e(E,[2,40]),e(E,[2,42]),e(E,[2,43]),e(E,[2,44]),e(E,[2,45]),e(E,[2,46]),e(E,[2,47]),e(E,[2,48]),e(E,[2,49]),e(E,[2,50]),e(E,[2,51]),e(w,[2,10]),e(w,[2,22],{30:41,29:56,24:z,27:lt}),e(w,[2,24]),e(w,[2,25]),{31:[1,57]},{11:59,32:58,37:24,38:d,39:R,40:27,41:_,42:A,43:m,44:T,45:S,46:P,47:p,48:C,49:h,50:L},e(w,[2,11]),e(w,[2,30],{33:60,27:ct}),e(w,[2,32]),{31:[1,61]},e(w,[2,12]),{17:62,24:$},{25:63,27:ut},e(w,[2,14]),{17:65,24:$},e(w,[2,16]),e(w,[2,17]),e(E,[2,41]),e(w,[2,23]),{27:[1,66]},{26:[1,67]},{26:[2,29],28:[1,68]},e(w,[2,31]),{27:[1,69]},e(w,[2,13]),{26:[1,70]},{26:[2,21],28:[1,71]},e(w,[2,15]),e(w,[2,26]),e(w,[2,27]),{11:59,32:72,37:24,38:d,39:R,40:27,41:_,42:A,43:m,44:T,45:S,46:P,47:p,48:C,49:h,50:L},e(w,[2,33]),e(w,[2,19]),{25:73,27:ut},{26:[2,28]},{26:[2,20]}],defaultActions:{8:[2,1],10:[2,3],21:[2,2],72:[2,28],73:[2,20]},parseError:function(r,l){if(l.recoverable)this.trace(r);else{var u=new Error(r);throw u.hash=l,u}},parse:function(r){var l=this,u=[0],g=[],b=[null],a=[],F=this.table,x="",U=0,gt=0,Vt=2,xt=1,Bt=a.slice.call(arguments,1),k=Object.create(this.lexer),B={yy:{}};for(var J in this.yy)Object.prototype.hasOwnProperty.call(this.yy,J)&&(B.yy[J]=this.yy[J]);k.setInput(r,B.yy),B.yy.lexer=k,B.yy.parser=this,typeof k.yylloc>"u"&&(k.yylloc={});var tt=k.yylloc;a.push(tt);var Wt=k.options&&k.options.ranges;typeof B.yy.parseError=="function"?this.parseError=B.yy.parseError:this.parseError=Object.getPrototypeOf(this).parseError;function Ot(){var I;return I=g.pop()||k.lex()||xt,typeof I!="number"&&(I instanceof Array&&(g=I,I=g.pop()),I=l.symbols_[I]||I),I}for(var D,W,v,it,O={},q,M,dt,j;;){if(W=u[u.length-1],this.defaultActions[W]?v=this.defaultActions[W]:((D===null||typeof D>"u")&&(D=Ot()),v=F[W]&&F[W][D]),typeof v>"u"||!v.length||!v[0]){var et="";j=[];for(q in F[W])this.terminals_[q]&&q>Vt&&j.push("'"+this.terminals_[q]+"'");k.showPosition?et="Parse error on line "+(U+1)+`:
 `+k.showPosition()+`
 Expecting `+j.join(", ")+", got '"+(this.terminals_[D]||D)+"'":et="Parse error on line "+(U+1)+": Unexpected "+(D==xt?"end of input":"'"+(this.terminals_[D]||D)+"'"),this.parseError(et,{text:k.match,token:this.terminals_[D]||D,line:k.yylineno,loc:tt,expected:j})}if(v[0]instanceof Array&&v.length>1)throw new Error("Parse Error: multiple actions possible at state: "+W+", token: "+D);switch(v[0]){case 1:u.push(D),b.push(k.yytext),a.push(k.yylloc),u.push(v[1]),D=null,gt=k.yyleng,x=k.yytext,U=k.yylineno,tt=k.yylloc;break;case 2:if(M=this.productions_[v[1]][1],O.$=b[b.length-M],O._$={first_line:a[a.length-(M||1)].first_line,last_line:a[a.length-1].last_line,first_column:a[a.length-(M||1)].first_column,last_column:a[a.length-1].last_column},Wt&&(O._$.range=[a[a.length-(M||1)].range[0],a[a.length-1].range[1]]),it=this.performAction.apply(O,[x,gt,U,B.yy,v[1],b,a].concat(Bt)),typeof it<"u")return it;M&&(u=u.slice(0,-1*M*2),b=b.slice(0,-1*M),a=a.slice(0,-1*M)),u.push(this.productions_[v[1]][0]),b.push(O.$),a.push(O._$),dt=F[u[u.length-2]][u[u.length-1]],u.push(dt);break;case 3:return!0}}return!0}},It=function(){var V={EOF:1,parseError:function(l,u){if(this.yy.parser)this.yy.parser.parseError(l,u);else throw new Error(l)},setInput:function(r,l){return this.yy=l||this.yy||{},this._input=r,this._more=this._backtrack=this.done=!1,this.yylineno=this.yyleng=0,this.yytext=this.matched=this.match="",this.conditionStack=["INITIAL"],this.yylloc={first_line:1,first_column:0,last_line:1,last_column:0},this.options.ranges&&(this.yylloc.range=[0,0]),this.offset=0,this},input:function(){var r=this._input[0];this.yytext+=r,this.yyleng++,this.offset++,this.match+=r,this.matched+=r;var l=r.match(/(?:\r\n?|\n).*/g);return l?(this.yylineno++,this.yylloc.last_line++):this.yylloc.last_column++,this.options.ranges&&this.yylloc.range[1]++,this._input=this._input.slice(1),r},unput:function(r){var l=r.length,u=r.split(/(?:\r\n?|\n)/g);this._input=r+this._input,this.yytext=this.yytext.substr(0,this.yytext.length-l),this.offset-=l;var g=this.match.split(/(?:\r\n?|\n)/g);this.match=this.match.substr(0,this.match.length-1),this.matched=this.matched.substr(0,this.matched.length-1),u.length-1&&(this.yylineno-=u.length-1);var b=this.yylloc.range;return this.yylloc={first_line:this.yylloc.first_line,last_line:this.yylineno+1,first_column:this.yylloc.first_column,last_column:u?(u.length===g.length?this.yylloc.first_column:0)+g[g.length-u.length].length-u[0].length:this.yylloc.first_column-l},this.options.ranges&&(this.yylloc.range=[b[0],b[0]+this.yyleng-l]),this.yyleng=this.yytext.length,this},more:function(){return this._more=!0,this},reject:function(){if(this.options.backtrack_lexer)this._backtrack=!0;else return this.parseError("Lexical error on line "+(this.yylineno+1)+`. You can only invoke reject() in the lexer when the lexer is of the backtracking persuasion (options.backtrack_lexer = true).
 `+this.showPosition(),{text:"",token:null,line:this.yylineno});return this},less:function(r){this.unput(this.match.slice(r))},pastInput:function(){var r=this.matched.substr(0,this.matched.length-this.match.length);return(r.length>20?"...":"")+r.substr(-20).replace(/\n/g,"")},upcomingInput:function(){var r=this.match;return r.length<20&&(r+=this._input.substr(0,20-r.length)),(r.substr(0,20)+(r.length>20?"...":"")).replace(/\n/g,"")},showPosition:function(){var r=this.pastInput(),l=new Array(r.length+1).join("-");return r+this.upcomingInput()+`

rasa/core/channels/inspector/dist/index.html

@@ -6,7 +6,7 @@
   <meta name="viewport" content="width=device-width, initial-scale=1.0" />
   <link rel="icon" href="data:image/x-icon;base64,AAABAAEAICAQAAEABADoAgAAFgAAACgAAAAgAAAAQAAAAAEABAAAAAAAAAIAAOwAAADsAAAAEAAAABAAAABxaUwAwoyZAMOJmADBi5kAwoyZAMKMmQDCjJkAwoyZAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAADQAAAAAAAAAAAAAAAAAACV3AAAAAAAAAAAAAAAAAAYXdwAAAAAAAAAAAAAAAANHd3cAAAAAAAEREREREREXd3d3ERAAAAAHd3d3d3d3d3FXd3dwAAAAB3d3d3d3d3dDB3d3cAAAAAd3IiIiIiIiAAIid3AAAAAHdwAAAAAAAAAAAHdwAAAAB3cAAAAAAAAAAAB3cAAAAAd3AAd3AABnd0AAd3AAAAAHdwAHdwADF3QgAHdwAAAAB3cAB3dTMXcSAAB3cAAAAAd3AAd3dxdxMAAAd3AAAAAHdwAHd3d3dGAAAHdwAAAAB3cAB3dHd3dxUAB3cAAAAAd3AAd3A1F3d3AAd3AAAAAHdwAHdwAANXdwAHdwAAAAB3cAB3cAAAAXcAB3cAAAAAd3AAd3d3d3d3AAd3AAAAAHdwAHd3d3d3dwAHdwAAAAB3cAB3d3d3d3cAB3cAAAAAd3AAAAAAAAAAAAd3AAAAAHdwAAAAAAAAAAAHdwAAAAB3cAAAAAAAAAAAB3cAAAAAd3d3d3d3d3d3d3d3AAAAAHd3d3d3d3d3d3d3dwAAAAB3d3d3d3d3d3d3d3cAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAD///5////4f///8H///8B/8AAAD/AAAA/wAAQP8AAcD/H//4/x//+P8ceDj/HHA4/xwAeP8cAPj/HAD4/xwAOP8cQDj/HHg4/xx+OP8cADj/HAA4/xwAOP8f//j/H//4/x//+P8AAAD/AAAA/wAAAP/////////////////////w==" sizes="32x32">
   <title>Rasa inspector</title>
-  <script type="module" crossorigin src="./assets/index-efdd30c1.js"></script>
+  <script type="module" crossorigin src="./assets/index-e7cef9de.js"></script>
   <link rel="stylesheet" href="./assets/index-3ee28881.css">
 </head>
 

rasa/core/channels/inspector/src/App.tsx

@@ -180,7 +180,7 @@ export function App() {
     });
   };
 
-  if (!rasaChatSessionId) return <LoadingSpinner />;
+  if (!rasaChatSessionId && !window.location.href.includes("socketio")) return <LoadingSpinner />;
 
   return (
     <Grid sx={gridSx}>

rasa/core/channels/inspector/src/helpers/audiostream.ts

@@ -8,6 +8,25 @@ const audioOptions = {
   }
 }
 
+const arrayBufferToBase64 = ( buffer: ArrayBuffer ): string => {
+    let binary = '';
+    const bytes = new Uint8Array( buffer );
+    const len = bytes.byteLength;
+    for (let i = 0; i < len; i++) {
+        binary += String.fromCharCode( bytes[ i ] );
+    }
+    return window.btoa( binary );
+}
+
+const base64ToArrayBuffer = ( s: string ): ArrayBuffer => {
+    const binary_string =  window.atob(s);
+    const len = binary_string.length;
+    const bytes = new Uint8Array( len );
+    for (let i = 0; i < len; i++)        {
+        bytes[i] = binary_string.charCodeAt(i);
+    }
+    return bytes.buffer;
+}
 
 const floatToIntArray = (arr: Float32Array): Int32Array => {
   // Convert Float Array [-1, 1] to full range int array
@@ -18,17 +37,29 @@ const intToFloatArray = (arr: Int32Array): Float32Array => {
   return Float32Array.from(arr, x => (x / 0x7fffffff))
 }
 
+interface Mark {
+    id: string
+    bytesToGo: number
+}
+
 interface AudioQueue {
   buffer: Float32Array;
+  marks: Array<Mark>
+  socket: WebSocket,
   write: (newAudio: Float32Array) => void;
   read: (nSamples: number) => Float32Array;
   length: () => number;
+  addMarker: (id: string) => void;
+  reduceMarkers: (bytesRead: number) => void;
+  popMarkers: () => void;
 }
 
 
-const createAudioQueue = () : AudioQueue => {
+const createAudioQueue = (socket: WebSocket) : AudioQueue => {
   return {
     buffer: new Float32Array(0),
+    marks: new Array<Mark>(),
+    socket,
 
     write: function(newAudio: Float32Array) {
       const currentQLength = this.buffer.length;
@@ -41,12 +72,42 @@ const createAudioQueue = () : AudioQueue => {
     read: function(nSamples: number) {
       const samplesToPlay = this.buffer.subarray(0, nSamples);
       this.buffer = this.buffer.subarray(nSamples, this.buffer.length);
+      this.reduceMarkers(samplesToPlay.length)
+      this.popMarkers()
       return samplesToPlay;
     },
 
     length: function() {
       return this.buffer.length;
+    },
+
+    addMarker: function(id: string) {
+        this.marks.push({id, bytesToGo: this.length()})
+    },
+
+    reduceMarkers: function(bytesRead: number) {
+        this.marks = this.marks.map((m) => {
+            return {id: m.id, bytesToGo: m.bytesToGo - bytesRead}
+        })
+    },
+
+    popMarkers: function() {
+        // marks are ordered
+        let popUpTo = 0;
+        while (popUpTo < this.marks.length) {
+            if (this.marks[popUpTo].bytesToGo <= 0) {
+                popUpTo += 1
+            } else {
+                break
+            }
+        }
+        const marksToPop = this.marks.slice(0, popUpTo)
+        this.marks = this.marks.slice(popUpTo, this.marks.length)
+        marksToPop.forEach((m) => {
+            this.socket.send(JSON.stringify({marker: m.id}))
+        })
     }
+
   };
 }
 
@@ -59,7 +120,10 @@ const streamMicrophoneToServer = async (socket: WebSocket) => {
     const audioInput = audioContext.createMediaStreamSource(audioStream)
     const sender = audioContext.createScriptProcessor(bufferSize, 1, 1)
     sender.onaudioprocess = function(event) {
-      socket.send(floatToIntArray(event.inputBuffer.getChannelData(0)))
+      const message = JSON.stringify({
+        "audio": arrayBufferToBase64(floatToIntArray(event.inputBuffer.getChannelData(0)).buffer)
+      })
+      socket.send(message)
     }
     audioInput.connect(sender)
     sender.connect(audioContext.destination)
@@ -68,8 +132,8 @@ const streamMicrophoneToServer = async (socket: WebSocket) => {
   }
 }
 
-const setupAudioPlayback = (): AudioQueue => {
-  const audioQueue = createAudioQueue()
+const setupAudioPlayback = (socket: WebSocket): AudioQueue => {
+  const audioQueue = createAudioQueue(socket)
   const silence = new Float32Array(bufferSize)
   const audioOutputContext = new AudioContext({sampleRate})
   const scriptNode = audioOutputContext.createScriptProcessor(bufferSize, 1, 1);
@@ -82,23 +146,20 @@ const setupAudioPlayback = (): AudioQueue => {
 }
 
 const addDataToAudioQueue = (audioQueue: AudioQueue) => (message: MessageEvent<any>) => {
-  if (message.data instanceof Blob) {
-    const reader = new FileReader();
-    reader.onload = function () {
-      const result = reader.result
-      if (result && result instanceof ArrayBuffer) {
-        const audioData = intToFloatArray(new Int32Array(result))
-        audioQueue.write(audioData);
-      }
-    };
-    reader.readAsArrayBuffer(message.data);
-  }
+    const data = JSON.parse(message.data.toString())
+    if (data["audio"]) {
+      const audioBytes = base64ToArrayBuffer(data["audio"])
+      const audioData = intToFloatArray(new Int32Array(audioBytes))
+      audioQueue.write(audioData);
+    } else if (data["marker"]) {
+        audioQueue.addMarker(data["marker"])
+    }
 }
 
 export async function createAudioConnection() {
   const websocketURL  = "ws://localhost:5005/webhooks/browser_audio/websocket"
   const socket = new WebSocket(websocketURL)
   socket.onopen = async () => { await streamMicrophoneToServer(socket)}
-  const audioQueue = setupAudioPlayback()
+  const audioQueue = setupAudioPlayback(socket)
   socket.onmessage = addDataToAudioQueue(audioQueue)
 }

rasa/core/channels/socketio.py

@@ -51,6 +51,7 @@ class SocketIOOutput(OutputChannel):
         return "socketio"
 
     def __init__(self, sio: AsyncServer, bot_message_evt: Text) -> None:
+        super().__init__()
         self.sio = sio
         self.bot_message_evt = bot_message_evt
         self.last_event_timestamp = (
@@ -59,7 +60,7 @@ class SocketIOOutput(OutputChannel):
 
     def _get_new_events(self) -> List[Dict[Text, Any]]:
         """Get events that are newer than the last sent event."""
-        events = self.tracker_state.get("events", [])
+        events = self.tracker_state.get("events", []) if self.tracker_state else []
         new_events = [
             event for event in events if event["timestamp"] > self.last_event_timestamp
         ]

rasa/core/channels/telegram.py

@@ -34,7 +34,7 @@ class TelegramOutput(Bot, OutputChannel):
         return "telegram"
 
     def __init__(self, access_token: Optional[Text]) -> None:
-        super().__init__(access_token)
+        Bot.__init__(self, access_token)
 
     async def send_text_message(
         self, recipient_id: Text, text: Text, **kwargs: Any

rasa/core/channels/twilio.py

@@ -28,7 +28,7 @@ class TwilioOutput(Client, OutputChannel):
         auth_token: Optional[Text],
         twilio_number: Optional[Text],
     ) -> None:
-        super().__init__(account_sid, auth_token)
+        Client.__init__(self, account_sid, auth_token)
         self.twilio_number = twilio_number
         self.send_retry = 0
         self.max_retry = 5

rasa/core/channels/voice_ready/jambonz.py

@@ -13,7 +13,7 @@ from sanic import Blueprint, response, Websocket  # type: ignore[attr-defined]
 from sanic.request import Request
 from sanic.response import HTTPResponse
 
-from rasa.shared.utils.common import mark_as_experimental_feature
+from rasa.shared.utils.common import mark_as_beta_feature
 
 
 structlogger = structlog.get_logger()
@@ -36,7 +36,7 @@ class JambonzVoiceReadyInput(InputChannel):
 
     def __init__(self) -> None:
         """Initializes the JambonzVoiceReadyInput channel."""
-        mark_as_experimental_feature("Jambonz Channel")
+        mark_as_beta_feature("Jambonz Channel")
         validate_voice_license_scope()
 
     def blueprint(

rasa/core/channels/voice_stream/asr/asr_event.py

@@ -11,3 +11,8 @@ class ASREvent:
 @dataclass
 class NewTranscript(ASREvent):
     text: str
+
+
+@dataclass
+class UserStartedSpeaking(ASREvent):
+    pass

rasa/core/channels/voice_stream/asr/azure.py

@@ -0,0 +1,122 @@
+import os
+from dataclasses import dataclass
+from typing import Any, Dict, Optional, AsyncIterator
+import asyncio
+
+from rasa.core.channels.voice_stream.asr.asr_engine import ASREngine, ASREngineConfig
+from rasa.core.channels.voice_stream.asr.asr_event import (
+    ASREvent,
+    NewTranscript,
+    UserStartedSpeaking,
+)
+from rasa.core.channels.voice_stream.audio_bytes import HERTZ, RasaAudioBytes
+from rasa.shared.exceptions import ConnectionException
+
+
+@dataclass
+class AzureASRConfig(ASREngineConfig):
+    language: Optional[str] = None
+    speech_region: Optional[str] = None
+
+
+class AzureASR(ASREngine[AzureASRConfig]):
+    def __init__(self, config: Optional[AzureASRConfig] = None):
+        import azure.cognitiveservices.speech as speechsdk
+
+        super().__init__(config)
+        self.speech_recognizer: Optional[speechsdk.SpeechRecognizer] = None
+        self.stream: Optional[speechsdk.audio.PushAudioInputStream] = None
+        self.is_recognizing = False
+        self.queue: asyncio.Queue[speechsdk.SpeechRecognitionEventArgs] = (
+            asyncio.Queue()
+        )
+
+    def signal_user_started_speaking(self, event: Any) -> None:
+        """Replace the unspecific azure event with a specific start event."""
+        self.fill_queue(UserStartedSpeaking())
+
+    def fill_queue(self, event: Any) -> None:
+        """Either puts the event or a dedicated ASR Event into the queue."""
+        self.queue.put_nowait(event)
+
+    async def connect(self) -> None:
+        import azure.cognitiveservices.speech as speechsdk
+
+        speech_config = speechsdk.SpeechConfig(
+            subscription=os.environ["AZURE_SPEECH_API_KEY"],
+            region=self.config.speech_region,
+        )
+        audio_format = speechsdk.audio.AudioStreamFormat(
+            samples_per_second=HERTZ,
+            bits_per_sample=8,
+            channels=1,
+            wave_stream_format=speechsdk.AudioStreamWaveFormat.MULAW,
+        )
+        self.stream = speechsdk.audio.PushAudioInputStream(stream_format=audio_format)
+        audio_config = speechsdk.audio.AudioConfig(stream=self.stream)
+        self.speech_recognizer = speechsdk.SpeechRecognizer(
+            speech_config=speech_config,
+            language=self.config.language,
+            audio_config=audio_config,
+        )
+        self.speech_recognizer.recognized.connect(self.fill_queue)
+        self.speech_recognizer.speech_start_detected.connect(
+            self.signal_user_started_speaking
+        )
+        self.speech_recognizer.start_continuous_recognition_async()
+        self.is_recognizing = True
+
+    async def close_connection(self) -> None:
+        if self.speech_recognizer is None:
+            raise ConnectionException("Websocket not connected.")
+        self.speech_recognizer.stop_continuous_recognition_async()
+
+    async def signal_audio_done(self) -> None:
+        """Signal to the ASR Api that you are done sending data."""
+        self.is_recognizing = False
+
+    def rasa_audio_bytes_to_engine_bytes(self, chunk: RasaAudioBytes) -> bytes:
+        """Convert RasaAudioBytes to bytes usable by this engine."""
+        return chunk
+
+    async def send_audio_chunks(self, chunk: RasaAudioBytes) -> None:
+        """Send audio chunks to the ASR system via the websocket."""
+        if self.speech_recognizer is None or self.stream is None:
+            raise ConnectionException("ASR not connected.")
+        engine_bytes = self.rasa_audio_bytes_to_engine_bytes(chunk)
+        self.stream.write(engine_bytes)
+
+    async def stream_asr_events(self) -> AsyncIterator[ASREvent]:
+        """Stream the events returned by the ASR system as it is fed audio bytes."""
+        if self.speech_recognizer is None:
+            raise ConnectionException("Websocket not connected.")
+        while self.is_recognizing or not self.queue.empty():
+            try:
+                message = await asyncio.wait_for(self.queue.get(), timeout=2)
+                asr_event = self.engine_event_to_asr_event(message)
+                if asr_event:
+                    yield asr_event
+            except asyncio.TimeoutError:
+                pass
+
+    def engine_event_to_asr_event(self, e: Any) -> Optional[ASREvent]:
+        """Translate an engine event to a common ASREvent."""
+        import azure.cognitiveservices.speech as speechsdk
+
+        if isinstance(e, speechsdk.SpeechRecognitionEventArgs) and isinstance(
+            e.result, speechsdk.SpeechRecognitionResult
+        ):
+            return NewTranscript(e.result.text)
+        if isinstance(e, ASREvent):
+            # transformation happened before
+            return e
+
+        return None
+
+    @staticmethod
+    def get_default_config() -> AzureASRConfig:
+        return AzureASRConfig("en-US", "germanywestcentral")
+
+    @classmethod
+    def from_config_dict(cls, config: Dict) -> "AzureASR":
+        return AzureASR(AzureASRConfig.from_dict(config))

rasa/core/channels/voice_stream/asr/deepgram.py

@@ -7,8 +7,12 @@ import websockets
 from websockets.legacy.client import WebSocketClientProtocol
 
 from rasa.core.channels.voice_stream.asr.asr_engine import ASREngine, ASREngineConfig
-from rasa.core.channels.voice_stream.asr.asr_event import ASREvent, NewTranscript
-from rasa.core.channels.voice_stream.audio_bytes import RasaAudioBytes
+from rasa.core.channels.voice_stream.asr.asr_event import (
+    ASREvent,
+    NewTranscript,
+    UserStartedSpeaking,
+)
+from rasa.core.channels.voice_stream.audio_bytes import HERTZ, RasaAudioBytes
 
 DEEPGRAM_API_KEY = "DEEPGRAM_API_KEY"
 
@@ -18,6 +22,9 @@ class DeepgramASRConfig(ASREngineConfig):
     endpoint: Optional[str] = None
     # number of miliseconds of silence to determine end of speech
     endpointing: Optional[int] = None
+    language: Optional[str] = None
+    model: Optional[str] = None
+    smart_format: Optional[bool] = None
 
 
 class DeepgramASR(ASREngine[DeepgramASRConfig]):
@@ -27,7 +34,7 @@ class DeepgramASR(ASREngine[DeepgramASRConfig]):
 
     async def open_websocket_connection(self) -> WebSocketClientProtocol:
         """Connect to the ASR system."""
-        deepgram_api_key = os.environ.get(DEEPGRAM_API_KEY)
+        deepgram_api_key = os.environ[DEEPGRAM_API_KEY]
         extra_headers = {"Authorization": f"Token {deepgram_api_key}"}
         api_url = self._get_api_url()
         query_params = self._get_query_params()
@@ -41,8 +48,9 @@ class DeepgramASR(ASREngine[DeepgramASRConfig]):
 
     def _get_query_params(self) -> str:
         return (
-            f"encoding=mulaw&sample_rate=8000&endpointing={self.config.endpointing}"
-            f"&vad_events=true"
+            f"encoding=mulaw&sample_rate={HERTZ}&endpointing={self.config.endpointing}"
+            f"&vad_events=true&language={self.config.language}"
+            f"&model={self.config.model}&smart_format={str(self.config.smart_format).lower()}"
         )
 
     async def signal_audio_done(self) -> None:
@@ -66,11 +74,13 @@ class DeepgramASR(ASREngine[DeepgramASRConfig]):
                 return NewTranscript(full_transcript)
             else:
                 self.accumulated_transcript += transcript
+        elif data.get("type") == "SpeechStarted":
+            return UserStartedSpeaking()
         return None
 
     @staticmethod
     def get_default_config() -> DeepgramASRConfig:
-        return DeepgramASRConfig("api.deepgram.com", 400)
+        return DeepgramASRConfig("api.deepgram.com", 400, "en", "nova-2-general", True)
 
     @classmethod
     def from_config_dict(cls, config: Dict) -> "DeepgramASR":

rasa/core/channels/voice_stream/audio_bytes.py

@@ -5,3 +5,4 @@ from typing import NewType
 # and tts engines
 # currently corresponds to raw wave, 8khz, 8bit, mono channel, mulaw encoding
 RasaAudioBytes = NewType("RasaAudioBytes", bytes)
+HERTZ = 8000

rasa/core/channels/voice_stream/browser_audio.py

@@ -1,7 +1,10 @@
 import audioop
+import base64
+import json
+
 import structlog
 import uuid
-from typing import Any, Awaitable, Callable, List, Optional
+from typing import Any, Awaitable, Callable, Optional, Tuple
 
 from sanic import Blueprint, HTTPResponse, Request, response
 from sanic import Websocket  # type: ignore
@@ -9,16 +12,19 @@ from sanic import Websocket  # type: ignore
 
 from rasa.core.channels import UserMessage
 from rasa.core.channels.voice_ready.utils import CallParameters
+from rasa.core.channels.voice_stream.call_state import call_state
 from rasa.core.channels.voice_stream.tts.tts_engine import TTSEngine
 from rasa.core.channels.voice_stream.audio_bytes import RasaAudioBytes
 from rasa.core.channels.voice_stream.voice_channel import (
+    ContinueConversationAction,
+    EndConversationAction,
     NewAudioAction,
     VoiceChannelAction,
     VoiceInputChannel,
     VoiceOutputChannel,
 )
 
-structlogger = structlog.get_logger()
+logger = structlog.get_logger()
 
 
 class BrowserAudioOutputChannel(VoiceOutputChannel):
@@ -31,10 +37,12 @@ class BrowserAudioOutputChannel(VoiceOutputChannel):
     ) -> bytes:
         return audioop.ulaw2lin(rasa_audio_bytes, 4)
 
-    def channel_bytes_to_messages(
-        self, recipient_id: str, channel_bytes: bytes
-    ) -> List[Any]:
-        return [channel_bytes]
+    def channel_bytes_to_message(self, recipient_id: str, channel_bytes: bytes) -> str:
+        return json.dumps({"audio": base64.b64encode(channel_bytes).decode("utf-8")})
+
+    def create_marker_message(self, recipient_id: str) -> Tuple[str, str]:
+        message_id = uuid.uuid4().hex
+        return json.dumps({"marker": message_id}), message_id
 
 
 class BrowserAudioInputChannel(VoiceInputChannel):
@@ -55,8 +63,23 @@ class BrowserAudioInputChannel(VoiceInputChannel):
         self,
         message: Any,
     ) -> VoiceChannelAction:
-        audio_bytes = self.channel_bytes_to_rasa_audio_bytes(message)
-        return NewAudioAction(audio_bytes)
+        data = json.loads(message)
+        if "audio" in data:
+            channel_bytes = base64.b64decode(data["audio"])
+            audio_bytes = self.channel_bytes_to_rasa_audio_bytes(channel_bytes)
+            return NewAudioAction(audio_bytes)
+        elif "marker" in data:
+            if data["marker"] == call_state.latest_bot_audio_id:
+                # Just finished streaming last audio bytes
+                call_state.is_bot_speaking = False  # type: ignore[attr-defined]
+                if call_state.should_hangup:
+                    logger.debug(
+                        "browser_audio.hangup", marker=call_state.latest_bot_audio_id
+                    )
+                    return EndConversationAction()
+            else:
+                call_state.is_bot_speaking = True  # type: ignore[attr-defined]
+        return ContinueConversationAction()
 
     def create_output_channel(
         self, voice_websocket: Websocket, tts_engine: TTSEngine

rasa/core/channels/voice_stream/call_state.py

@@ -0,0 +1,23 @@
+import asyncio
+from contextvars import ContextVar
+from werkzeug.local import LocalProxy
+from dataclasses import dataclass
+from typing import Optional
+
+
+# Per voice session data
+# This is similar to how flask makes the "request" object available as a global variable
+# It's a "global" variable that is local to an async task (i.e. websocket session)
+@dataclass
+class CallState:
+    is_user_speaking: bool = False
+    is_bot_speaking: bool = False
+    silence_timeout_watcher: Optional[asyncio.Task] = None
+    silence_timeout: Optional[float] = None
+    latest_bot_audio_id: Optional[str] = None
+    should_hangup: bool = False
+    connection_failed: bool = False
+
+
+_call_state: ContextVar[CallState] = ContextVar("call_state")
+call_state = LocalProxy(_call_state)

rasa/core/channels/voice_stream/tts/azure.py

@@ -4,7 +4,7 @@ from dataclasses import dataclass
 
 import aiohttp
 import structlog
-from aiohttp import ClientConnectorError
+from aiohttp import ClientConnectorError, ClientTimeout
 
 from rasa.core.channels.voice_stream.audio_bytes import RasaAudioBytes
 from rasa.core.channels.voice_stream.tts.tts_engine import (
@@ -28,10 +28,11 @@ class AzureTTS(TTSEngine[AzureTTSConfig]):
 
     def __init__(self, config: Optional[AzureTTSConfig] = None):
         super().__init__(config)
+        timeout = ClientTimeout(total=self.config.timeout)
         # Have to create this class-shared session lazily at run time otherwise
         # the async event loop doesn't work
         if self.__class__.session is None or self.__class__.session.closed:
-            self.__class__.session = aiohttp.ClientSession()
+            self.__class__.session = aiohttp.ClientSession(timeout=timeout)
 
     async def synthesize(
         self, text: str, config: Optional[AzureTTSConfig] = None
@@ -60,6 +61,8 @@ class AzureTTS(TTSEngine[AzureTTSConfig]):
                     raise TTSError(f"TTS failed: {response.text()}")
         except ClientConnectorError as e:
             raise TTSError(e)
+        except TimeoutError as e:
+            raise TTSError(e)
 
     @staticmethod
     def get_request_headers() -> dict[str, str]:
@@ -92,6 +95,7 @@ class AzureTTS(TTSEngine[AzureTTSConfig]):
         return AzureTTSConfig(
             language="en-US",
             voice="en-US-JennyNeural",
+            timeout=10,
             speech_region="germanywestcentral",
         )