sip-lab 1.27.1 → 1.28.0

package/README.md

@@ -14,10 +14,10 @@ It permits to:
   - send/receive audio using SRTP
   - do speech synth using flite
   - do speech recog using pocketsphinx (but only works well with sampling rate of 16000)
+  - do speech synth/recog using [ws_speech_server](https://github.com/MayamaTakeshi/ws_speech_server) (this permits to use google/amazon/azure/etc speech services)
 
 TODO:
   - add support for video playing/recording from/to file
-  - add support for speech synth/recog via websocket server to permit to access Google Speech, Whisper, Amazon Poly etc.
   - add support for T.38 fax
   - add support for SIP over WebSocket
   - add support for WebRTC
@@ -47,6 +47,34 @@ The above script has detailed comments.
 
 Please read it to undestand how to write your own test scripts.
 
+
+### Samples
+
+See general sample scripts in folder samples.
+
+There are additional samples scripts in folder samples_extra but they require [ws_speech_server](https://github.com/MayamaTakeshi/ws_speech_server) to be running locally (and it should be started with GOOGLE_APPLICATION_CREDENTIALS set).
+
+To run ws_speech_server, do this:
+```
+https://github.com/MayamaTakeshi/ws_speech_server
+cd ws_speech_server
+npm i
+npm run build
+cp config/default.js.sample config/default.js
+export GOOGLE_APPLICATION_CREDENTIALS=/path/to/your/credentials/file
+node src/App.bs.js
+```
+
+Then you should be able to test with dtmf language:
+```
+node samples_extra/ws_speech_server.dtmf.js 
+```
+or with google speech service:
+```
+node samples_extra/ws_speech_server.google.js
+```
+
+
 ### About the code
 
 Although the code in written in *.cpp/*.hpp named files, this is not actually a C++ project.

package/binding.gyp

@@ -19,10 +19,12 @@
             "3rdParty/spandsp/src",
             "3rdParty/pocketsphinx/include",
             "3rdParty/pocketsphinx/build/include",
+            "3rdParty/pjwebsock/websock",
             "<!@(node -p \"require('node-addon-api').include\")",
           ],
           'conditions': [
             [ 'OS!="win"', {
+              'cflags': ['-g'],
               'cflags_cc': [
                 '-g',
                 '-fexceptions',
@@ -120,6 +122,12 @@
         'src/pjmedia/src/pjmedia/fax_port.c',
         'src/pjmedia/src/pjmedia/flite_port.c',
         'src/pjmedia/src/pjmedia/pocketsphinx_port.c',
+        'src/pjmedia/src/pjmedia/ws_speech_port.cpp',
+        '3rdParty/pjwebsock/websock/http.c',
+        '3rdParty/pjwebsock/websock/websock_transport_tcp.c',
+        '3rdParty/pjwebsock/websock/websock_transport_tls.c',
+        '3rdParty/pjwebsock/websock/websock.c',
+        '3rdParty/pjwebsock/websock/websock_transport.c',
       ],
     },
   ],

package/build_deps.sh

@@ -102,6 +102,15 @@ then
 fi
 
 
+cd $START_DIR/3rdParty
+if [[ ! -d pjwebsock ]]
+then
+    git clone https://github.com/jimying/pjwebsock
+    cd pjwebsock
+    git checkout a0616ea27f01d5e3bdfd5b801fb1499473a0b0cb
+fi
+
+
 #cd $START_DIR/3rdParty
 #if [[ ! -d openssl ]]
 #then

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "sip-lab",
-  "version": "1.27.1",
+  "version": "1.28.0",
   "description": "",
   "main": "index.js",
   "engines": {

package/samples/play_wav_and_speech_recog.bad_transcript.pcmu8000.js

@@ -115,7 +115,7 @@ async function test() {
     sip.call.start_speech_recog(oc.id)
     sip.call.start_speech_recog(ic.id)
 
-    await z.sleep(100)
+    await z.sleep(200)
 
     sip.call.start_play_wav(oc.id, {file: 'samples/artifacts/hello_good_morning.wav', end_of_file_event: true, no_loop: true})
     sip.call.start_play_wav(ic.id, {file: 'samples/artifacts/hello_good_morning.wav', end_of_file_event: true, no_loop: true})
@@ -130,16 +130,16 @@ async function test() {
             call_id: oc.id,
         },
         {
-            event: 'speech_transcript',
+            event: 'speech',
             call_id: oc.id,
             //transcript: 'hello good morning', // bad transcript (will not match)
         },
         {
-            event: 'speech_transcript',
+            event: 'speech',
             call_id: ic.id,
             //transcript: 'hello good morning', // bad transcript (will not match)
         },
-    ], 4000)
+    ], 5000)
 
     sip.call.stop_record_wav(oc.id)
     sip.call.stop_record_wav(ic.id)

package/samples/speech_synth_and_recog.speex16000.js

@@ -117,25 +117,25 @@ async function test() {
 
     await z.sleep(100)
 
-    sip.call.start_speech_synth(oc.id, {voice: 'kal16', text: 'Good morning.', end_of_speech_event: true, no_loop: true})
-    sip.call.start_speech_synth(ic.id, {voice: 'kal16', text: 'How are you?', end_of_speech_event: true, no_loop: true})
+    sip.call.start_speech_synth(oc.id, {voice: 'kal16', text: 'Good morning.'})
+    sip.call.start_speech_synth(ic.id, {voice: 'kal16', text: 'How are you?'})
 
     await z.wait([
         {
-            event: 'end_of_speech',
+            event: 'speech_synth_complete',
             call_id: ic.id,
         },
         {
-            event: 'end_of_speech',
+            event: 'speech_synth_complete',
             call_id: oc.id,
         },
         {
-            event: 'speech_transcript',
+            event: 'speech',
             call_id: oc.id,
             transcript: 'how are you',
         },
         {
-            event: 'speech_transcript',
+            event: 'speech',
             call_id: ic.id,
             transcript: 'good morning',
         },

package/samples/text_to_speech.js

@@ -130,26 +130,19 @@ async function test() {
 	},
     ], 3000)
 
-    sip.call.start_speech_synth(oc.id, {voice: 'slt', text: 'Hello World.', end_of_speech_event: true})
-    sip.call.start_speech_synth(ic.id, {voice: 'kal', text: 'How are you?', end_of_speech_event: true, no_loop: true})
+    sip.call.start_speech_synth(oc.id, {voice: 'slt', text: 'Hello World.'})
+    sip.call.start_speech_synth(ic.id, {voice: 'kal', text: 'How are you?'})
 
     await z.wait([
         {
-            event: 'end_of_speech',
+            event: 'speech_synth_complete',
             call_id: ic.id,
         },
         {
-            event: 'end_of_speech',
+            event: 'speech_synth_complete',
             call_id: oc.id,
         },
-    ], 2000)
-
-    await z.wait([
-        {
-            event: 'end_of_speech',
-            call_id: oc.id,
-        },
-    ], 2000)
+    ], 3000)
 
     sip.call.stop_speech_synth(oc.id) // this is not actually necessary. It is used just to confirm the command works
     sip.call.stop_speech_synth(ic.id) // this is not actually necessary. It is used just to confirm the command works

package/src/event_templates.cpp

@@ -105,16 +105,16 @@ int make_evt_end_of_file(char *dest, int size, long call_id) {
       "{\"event\": \"end_of_file\", \"call_id\": %ld}", call_id);
 }
 
-int make_evt_end_of_speech(char *dest, int size, long call_id) {
+int make_evt_speech_synth_complete(char *dest, int size, long call_id) {
   return snprintf(
       dest, size,
-      "{\"event\": \"end_of_speech\", \"call_id\": %ld}", call_id);
+      "{\"event\": \"speech_synth_complete\", \"call_id\": %ld}", call_id);
 }
 
-int make_evt_speech_transcript(char *dest, int size, long call_id, char* transcript) {
+int make_evt_speech(char *dest, int size, long call_id, char* transcript) {
   return snprintf(
       dest, size,
-      "{\"event\": \"speech_transcript\", \"call_id\": %ld, \"transcript\": \"%s\"}", call_id, transcript);
+      "{\"event\": \"speech\", \"call_id\": %ld, \"transcript\": \"%s\"}", call_id, transcript);
 }
 
 int make_evt_tcp_msg(char *dest, int size, long call_id, const char *protocol, char *data, int data_len) {
@@ -122,3 +122,10 @@ int make_evt_tcp_msg(char *dest, int size, long call_id, const char *protocol, c
       dest, size,
       "{\"event\": \"%s_msg\", \"call_id\": %ld}\n%.*s", protocol, call_id, data_len, data);
 }
+
+int make_evt_ws_speech_event(char *dest, int size, long call_id, char *data, int data_len) {
+  return snprintf(
+      dest, size,
+      "{\"event\": \"ws_speech_event\", \"call_id\": %ld, \"data\": %.*s}", call_id, data_len, data);
+}
+

package/src/event_templates.hpp

@@ -36,10 +36,12 @@ int make_evt_fax_result(char *dest, int size, long call_id, int result);
 
 int make_evt_end_of_file(char *dest, int size, long call_id);
 
-int make_evt_end_of_speech(char *dest, int size, long call_id);
+int make_evt_speech_synth_complete(char *dest, int size, long call_id);
 
-int make_evt_speech_transcript(char *dest, int size, long call_id, char* transcript);
+int make_evt_speech(char *dest, int size, long call_id, char* transcript);
 
 int make_evt_tcp_msg(char *dest, int size, long call_id, const char *protocol, char *data, int data_len);
 
+int make_evt_ws_speech_event(char *dest, int size, long call_id, char *data, int data_len);
+
 #endif

package/src/pjmedia/include/pjmedia/flite_port.h

@@ -5,7 +5,7 @@
 
 PJ_BEGIN_DECL
 
-enum pjmedia_filte_option
+enum pjmedia_flite_option
 {
   PJMEDIA_SPEECH_NO_LOOP = 1
 };
@@ -25,7 +25,7 @@ PJ_DEF(pj_status_t) pjmedia_flite_port_set_eof_cb(pjmedia_port *port,
 
 PJ_DEF(pj_status_t) pjmedia_flite_port_speak( pjmedia_port *port,
                                           const char *text,
-                                          unsigned options);
+                                          int times);
 
 PJ_END_DECL
 

package/src/pjmedia/include/pjmedia/ws_speech_port.h

@@ -0,0 +1,37 @@
+#ifndef __WS_SPEECH_PORT_H__
+#define __WS_SPEECH_PORT_H__
+
+#include <pjmedia/port.h>
+#include "websock.h"
+
+PJ_BEGIN_DECL
+
+enum ws_speech_event
+{
+  WS_SPEECH_EVENT_CONNECTED,
+  WS_SPEECH_EVENT_CONNECTION_ERROR,
+  WS_SPEECH_EVENT_DISCONNECTED,
+  WS_SPEECH_EVENT_TEXT_MSG
+};
+
+PJ_DEF(pj_status_t) pjmedia_ws_speech_port_create( pj_pool_t *pool,
+				unsigned clock_rate,
+				unsigned channel_count,
+				unsigned samples_per_frame,
+				unsigned bits_per_sample,
+                pj_websock_endpoint *ws_endpt,
+                const char *server_url,
+                const char *ss_engine,
+                const char *ss_voice,
+                const char *ss_language,
+                const char *ss_text,
+                int ss_times,
+                const char *sr_engine,
+                const char *sr_language,
+                void (*cb)(pjmedia_port*, void *user_data, enum ws_speech_event, char *data, int len),
+                void *cb_user_data,
+				pjmedia_port **p_port);
+
+PJ_END_DECL
+
+#endif	/* __WS_SPEECH_PORT_H__ */

package/src/pjmedia/src/pjmedia/flite_port.c

@@ -53,7 +53,6 @@ static struct {
 
 struct flite_t {
     struct pjmedia_port base;
-    unsigned         options;
 
 	cst_voice *v;
     unsigned written_samples;
@@ -61,6 +60,8 @@ struct flite_t {
 
     pj_bool_t        subscribed;
     void           (*cb)(pjmedia_port*, void*);
+
+    int times;
 };
 
 #define free_wave(w) if (w) {delete_wave(w) ; w = NULL; }
@@ -112,6 +113,7 @@ PJ_DEF(pj_status_t) pjmedia_flite_port_create( pj_pool_t *pool,
 				const char *voice,
 				pjmedia_port **p_port)
 {
+    printf("pjmedia_flite_port_create\n");
     struct flite_t *flite;
     const pj_str_t name = pj_str("flite_data");
 
@@ -164,13 +166,14 @@ PJ_DEF(pj_status_t) pjmedia_flite_port_create( pj_pool_t *pool,
 
 PJ_DEF(pj_status_t) pjmedia_flite_port_speak( pjmedia_port *port,
                                           const char *text,
-                                          unsigned options) {
+                                          int times) {
+    printf("pjmedia_flite_port_speak. text=%s times=%i\n", text, times);
     struct flite_t *flite = (struct flite_t*)port;
     if(flite->w) {
         free_wave(flite->w);
     }
     
-    flite->options = options;
+    flite->times = times;
 
     flite->w = flite_text_to_wave(text, flite->v);
     if ((unsigned)flite->w->sample_rate != PJMEDIA_PIA_SRATE(&port->info)) {
@@ -185,22 +188,25 @@ PJ_DEF(pj_status_t) pjmedia_flite_port_speak( pjmedia_port *port,
 // called when pjmedia needs data to be sent out
 static pj_status_t flite_get_frame(pjmedia_port *port,
 					pjmedia_frame *frame) {
+    printf("flite_get_frame\n");
 
 	PJ_ASSERT_RETURN(port && frame, PJ_EINVAL);
 
     struct flite_t *flite = (struct flite_t*)port;
 
-    if(!flite->w) {
-        //printf("flite no data\n");
+    if(flite->times <= 0 || !flite->w) {
+        printf("flite no data\n");
         frame->type = PJMEDIA_FRAME_TYPE_NONE;
         return PJ_SUCCESS;
     }
 
-    //printf("written_samples=%i num_samples=%i\n", flite->written_samples, flite->w->num_samples);
+    printf("written_samples=%i num_samples=%i\n", flite->written_samples, flite->w->num_samples);
     if (flite->written_samples + PJMEDIA_PIA_SPF(&port->info) > (unsigned)flite->w->num_samples) {
         printf("flite end of speech\n");
 
-        if(flite->cb) {
+        flite->times--;
+
+        if(flite->times <= 0 && flite->cb) {
             if (!flite->subscribed) {
                 pj_status_t status = pjmedia_event_subscribe(NULL, &speech_on_event,
                                                  flite, flite);
@@ -218,10 +224,9 @@ static pj_status_t flite_get_frame(pjmedia_port *port,
             }
         }
 
-        pj_bool_t no_loop = (flite->options & PJMEDIA_SPEECH_NO_LOOP);
-
-        if(no_loop) {
+        if(flite->times <= 0) {
             free_wave(flite->w);
+            flite->w = NULL;
             frame->type = PJMEDIA_FRAME_TYPE_NONE;
             return PJ_SUCCESS;
         } else {
@@ -232,7 +237,7 @@ static pj_status_t flite_get_frame(pjmedia_port *port,
     memcpy(frame->buf, flite->w->samples + flite->written_samples, PJMEDIA_PIA_SPF(&port->info)*2);
     flite->written_samples += PJMEDIA_PIA_SPF(&port->info);
     frame->type = PJMEDIA_FRAME_TYPE_AUDIO;
-    //printf("flite data written samples=%i\n", PJMEDIA_PIA_SPF(&port->info));
+    printf("flite data written samples=%i\n", PJMEDIA_PIA_SPF(&port->info));
 
     return PJ_SUCCESS;
 }