sip-lab 1.27.1 → 1.28.1

package/README.md

@@ -14,10 +14,10 @@ It permits to:
   - send/receive audio using SRTP
   - do speech synth using flite
   - do speech recog using pocketsphinx (but only works well with sampling rate of 16000)
+  - do speech synth/recog using [ws_speech_server](https://github.com/MayamaTakeshi/ws_speech_server) (this permits to use google/amazon/azure/etc speech services)
 
 TODO:
   - add support for video playing/recording from/to file
-  - add support for speech synth/recog via websocket server to permit to access Google Speech, Whisper, Amazon Poly etc.
   - add support for T.38 fax
   - add support for SIP over WebSocket
   - add support for WebRTC
@@ -40,13 +40,41 @@ npm install sip-lab
 
 Then run some sample script from subfolder samples:
 ```
-node samples/simple.js
+node node_modules/sip-lab/samples/simple.js
 ```
 
 The above script has detailed comments. 
 
 Please read it to undestand how to write your own test scripts.
 
+
+### Samples
+
+See general sample scripts in folder samples.
+
+There are additional samples scripts in folder samples_extra but they require [ws_speech_server](https://github.com/MayamaTakeshi/ws_speech_server) to be running locally (and it should be started with GOOGLE_APPLICATION_CREDENTIALS set).
+
+To run ws_speech_server, do this:
+```
+https://github.com/MayamaTakeshi/ws_speech_server
+cd ws_speech_server
+npm i
+npm run build
+cp config/default.js.sample config/default.js
+export GOOGLE_APPLICATION_CREDENTIALS=/path/to/your/credentials/file
+node src/App.bs.js
+```
+
+Then you should be able to test with dtmf language:
+```
+node node_modules/sip-lab/samples_extra/ws_speech_server.dtmf.js
+```
+or with google speech service:
+```
+node node_modules/sip-lab/samples_extra/ws_speech_server.google.js
+```
+
+
 ### About the code
 
 Although the code in written in *.cpp/*.hpp named files, this is not actually a C++ project.

package/binding.gyp

@@ -19,10 +19,12 @@
             "3rdParty/spandsp/src",
             "3rdParty/pocketsphinx/include",
             "3rdParty/pocketsphinx/build/include",
+            "3rdParty/pjwebsock/websock",
             "<!@(node -p \"require('node-addon-api').include\")",
           ],
           'conditions': [
             [ 'OS!="win"', {
+              'cflags': ['-g'],
               'cflags_cc': [
                 '-g',
                 '-fexceptions',
@@ -120,6 +122,12 @@
         'src/pjmedia/src/pjmedia/fax_port.c',
         'src/pjmedia/src/pjmedia/flite_port.c',
         'src/pjmedia/src/pjmedia/pocketsphinx_port.c',
+        'src/pjmedia/src/pjmedia/ws_speech_port.cpp',
+        '3rdParty/pjwebsock/websock/http.c',
+        '3rdParty/pjwebsock/websock/websock_transport_tcp.c',
+        '3rdParty/pjwebsock/websock/websock_transport_tls.c',
+        '3rdParty/pjwebsock/websock/websock.c',
+        '3rdParty/pjwebsock/websock/websock_transport.c',
       ],
     },
   ],

package/build_deps.sh

@@ -102,6 +102,15 @@ then
 fi
 
 
+cd $START_DIR/3rdParty
+if [[ ! -d pjwebsock ]]
+then
+    git clone https://github.com/jimying/pjwebsock
+    cd pjwebsock
+    git checkout a0616ea27f01d5e3bdfd5b801fb1499473a0b0cb
+fi
+
+
 #cd $START_DIR/3rdParty
 #if [[ ! -d openssl ]]
 #then

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "sip-lab",
-  "version": "1.27.1",
+  "version": "1.28.1",
   "description": "",
   "main": "index.js",
   "engines": {
@@ -44,6 +44,7 @@
     "README.md",
     "DEV.md",
     "samples",
+    "samples_extra",
     "prebuilds",
     "pocketsphinx",
     "runtests"

package/samples/play_wav_and_speech_recog.bad_transcript.pcmu8000.js

@@ -115,7 +115,7 @@ async function test() {
     sip.call.start_speech_recog(oc.id)
     sip.call.start_speech_recog(ic.id)
 
-    await z.sleep(100)
+    await z.sleep(200)
 
     sip.call.start_play_wav(oc.id, {file: 'samples/artifacts/hello_good_morning.wav', end_of_file_event: true, no_loop: true})
     sip.call.start_play_wav(ic.id, {file: 'samples/artifacts/hello_good_morning.wav', end_of_file_event: true, no_loop: true})
@@ -130,16 +130,16 @@ async function test() {
             call_id: oc.id,
         },
         {
-            event: 'speech_transcript',
+            event: 'speech',
             call_id: oc.id,
             //transcript: 'hello good morning', // bad transcript (will not match)
         },
         {
-            event: 'speech_transcript',
+            event: 'speech',
             call_id: ic.id,
             //transcript: 'hello good morning', // bad transcript (will not match)
         },
-    ], 4000)
+    ], 5000)
 
     sip.call.stop_record_wav(oc.id)
     sip.call.stop_record_wav(ic.id)

package/samples/speech_synth_and_recog.speex16000.js

@@ -117,25 +117,25 @@ async function test() {
 
     await z.sleep(100)
 
-    sip.call.start_speech_synth(oc.id, {voice: 'kal16', text: 'Good morning.', end_of_speech_event: true, no_loop: true})
-    sip.call.start_speech_synth(ic.id, {voice: 'kal16', text: 'How are you?', end_of_speech_event: true, no_loop: true})
+    sip.call.start_speech_synth(oc.id, {voice: 'kal16', text: 'Good morning.'})
+    sip.call.start_speech_synth(ic.id, {voice: 'kal16', text: 'How are you?'})
 
     await z.wait([
         {
-            event: 'end_of_speech',
+            event: 'speech_synth_complete',
             call_id: ic.id,
         },
         {
-            event: 'end_of_speech',
+            event: 'speech_synth_complete',
             call_id: oc.id,
         },
         {
-            event: 'speech_transcript',
+            event: 'speech',
             call_id: oc.id,
             transcript: 'how are you',
         },
         {
-            event: 'speech_transcript',
+            event: 'speech',
             call_id: ic.id,
             transcript: 'good morning',
         },

package/samples/text_to_speech.js

@@ -130,26 +130,19 @@ async function test() {
 	},
     ], 3000)
 
-    sip.call.start_speech_synth(oc.id, {voice: 'slt', text: 'Hello World.', end_of_speech_event: true})
-    sip.call.start_speech_synth(ic.id, {voice: 'kal', text: 'How are you?', end_of_speech_event: true, no_loop: true})
+    sip.call.start_speech_synth(oc.id, {voice: 'slt', text: 'Hello World.'})
+    sip.call.start_speech_synth(ic.id, {voice: 'kal', text: 'How are you?'})
 
     await z.wait([
         {
-            event: 'end_of_speech',
+            event: 'speech_synth_complete',
             call_id: ic.id,
         },
         {
-            event: 'end_of_speech',
+            event: 'speech_synth_complete',
             call_id: oc.id,
         },
-    ], 2000)
-
-    await z.wait([
-        {
-            event: 'end_of_speech',
-            call_id: oc.id,
-        },
-    ], 2000)
+    ], 3000)
 
     sip.call.stop_speech_synth(oc.id) // this is not actually necessary. It is used just to confirm the command works
     sip.call.stop_speech_synth(ic.id) // this is not actually necessary. It is used just to confirm the command works

package/samples_extra/ws_speech_server.dtmf.js

@@ -0,0 +1,194 @@
+var sip = require ('../index.js')
+var Zeq = require('@mayama/zeq')
+var z = new Zeq()
+var m = require('data-matching')
+var sip_msg = require('sip-matching')
+var sdp = require('sdp-matching')
+
+async function test() {
+    //sip.set_log_level(6)
+    sip.dtmf_aggregation_on(500)
+
+    sip.set_codecs("PCMU/8000/1:128")
+
+    z.trap_events(sip.event_source, 'event', (evt) => {
+        var e = evt.args[0]
+        return e
+    })
+
+    console.log(sip.start((data) => { console.log(data)} ))
+
+    t1 = sip.transport.create({address: "127.0.0.1", type: 'tcp'})
+    t2 = sip.transport.create({address: "127.0.0.1", type: 'tcp'})
+
+    console.log("t1", t1)
+    console.log("t2", t2)
+
+    oc = sip.call.create(t1.id, {
+        from_uri: '"abc"<sip:alice@test.com>',
+        to_uri: `sip:bob@${t2.address}:${t2.port}`,
+        headers: {
+            'X-MyHeader1': 'abc',
+            'X-MyHeader2': 'def',
+        },
+    })
+
+    await z.wait([
+        {
+            event: "incoming_call",
+            call_id: m.collect("call_id"),
+            msg: sip_msg({
+                $rm: 'INVITE',
+                $fU: 'alice',
+                $fd: 'test.com',
+                $tU: 'bob',
+                '$hdr(X-MyHeader1)': 'abc',
+                'hdr_x_myheader2': 'def',
+            }),
+        },
+        {
+            event: 'response',
+            call_id: oc.id,
+            method: 'INVITE',
+            msg: sip_msg({
+                $rs: '100',
+                $rr: 'Trying',
+                '$(hdrcnt(via))': 1,
+                'hdr_call_id': m.collect('sip_call_id'),
+                $fU: 'alice',
+                $fd: 'test.com',
+                $tU: 'bob',
+                '$hdr(l)': '0',
+            }),
+        },
+    ], 1000)
+
+    ic = {
+        id: z.store.call_id,
+        sip_call_id: z.store.sip_call_id,
+    }
+
+    sip.call.respond(ic.id, {
+        code: 200,
+        reason:'OK',
+        headers: {
+            'X-MyHeader3': 'ghi',
+            'X-MyHeader4': 'jkl',
+        },
+    })
+
+    await z.wait([
+        {
+            event: 'media_update',
+            call_id: oc.id,
+            status: 'ok',
+        },
+        {
+            event: 'media_update',
+            call_id: ic.id,
+            status: 'ok',
+        },
+        {
+            event: 'response',
+            call_id: oc.id,
+            method: 'INVITE',
+            msg: sip_msg({
+                $rs: '200',
+                $rr: 'OK',
+                '$(hdrcnt(v))': 1,
+                $fU: 'alice',
+                $fd: 'test.com',
+                $tU: 'bob',
+                '$hdr(content-type)': 'application/sdp',
+                $rb: '!{_}a=sendrecv',
+                '$hdr(X-MyHeader3)': 'ghi',
+                '$hdr(X-MyHeader4)': 'jkl',
+            }),
+        },
+    ], 1000)
+
+    sip.call.start_record_wav(oc.id, {file: './oc.wav'})
+    sip.call.start_record_wav(ic.id, {file: './ic.wav'})
+
+    sip.call.start_speech_synth(oc.id, {server_url: 'ws://0.0.0.0:8080', engine: 'dtmf-gen', voice: 'dtmf', language: 'dtmf', text: 'abcd', times: 1})
+    sip.call.start_speech_synth(ic.id, {server_url: 'ws://0.0.0.0:8080', engine: 'dtmf-gen', voice: 'dtmf', language: 'dtmf', text: 'dcba', times: 1})
+
+    sip.call.start_speech_recog(oc.id, {server_url: 'ws://0.0.0.0:8080', engine: 'dtmf-det', language: 'dtmf'})
+    sip.call.start_speech_recog(ic.id, {server_url: 'ws://0.0.0.0:8080', engine: 'dtmf-det', language: 'dtmf'})
+
+    await z.wait([
+        {
+            event: 'speech_synth_complete',
+            call_id: ic.id,
+        },
+        {
+            event: 'speech_synth_complete',
+            call_id: oc.id,
+        },
+        {
+            event: 'dtmf',
+            call_id: oc.id,
+            digits: 'dcba',
+            mode: 1,
+            media_id: 0
+        },
+        {
+            event: 'dtmf',
+            call_id: ic.id,
+            digits: 'abcd',
+            mode: 1,
+            media_id: 0
+        },
+        {
+            event: 'speech',
+            call_id: oc.id,
+            transcript: 'dcba'
+        },
+        {
+            event: 'speech',
+            call_id: ic.id,
+            transcript: 'abcd'
+        },
+    ], 3000)
+
+    await z.sleep(1000)
+
+    sip.call.stop_record_wav(oc.id)
+    sip.call.stop_record_wav(ic.id)
+
+    sip.call.terminate(oc.id)
+
+    await z.wait([
+        {
+            event: 'call_ended',
+            call_id: oc.id,
+        },
+        {
+            event: 'call_ended',
+            call_id: ic.id,
+        },
+        {
+            event: 'response',
+            call_id: oc.id,
+            method: 'BYE',
+            msg: sip_msg({
+                $rs: '200',
+                $rr: 'OK',
+            }),
+        },
+    ], 1000)
+
+    await z.sleep(100)
+
+    console.log("Success")
+
+    sip.stop()
+}
+
+
+test()
+.catch(e => {
+    console.error(e)
+    process.exit(1)
+})
+

package/samples_extra/ws_speech_server.google.js

@@ -0,0 +1,190 @@
+var sip = require ('../index.js')
+var Zeq = require('@mayama/zeq')
+var z = new Zeq()
+var m = require('data-matching')
+var sip_msg = require('sip-matching')
+var sdp = require('sdp-matching')
+
+async function test() {
+    //sip.set_log_level(6)
+    sip.dtmf_aggregation_on(500)
+
+    sip.set_codecs("PCMU/8000/1:128")
+
+    z.trap_events(sip.event_source, 'event', (evt) => {
+        var e = evt.args[0]
+        return e
+    })
+
+    console.log(sip.start((data) => { console.log(data)} ))
+
+    t1 = sip.transport.create({address: "127.0.0.1", type: 'tcp'})
+    t2 = sip.transport.create({address: "127.0.0.1", type: 'tcp'})
+
+    console.log("t1", t1)
+    console.log("t2", t2)
+
+    oc = sip.call.create(t1.id, {
+        from_uri: '"abc"<sip:alice@test.com>',
+        to_uri: `sip:bob@${t2.address}:${t2.port}`,
+        headers: {
+            'X-MyHeader1': 'abc',
+            'X-MyHeader2': 'def',
+        },
+    })
+
+    await z.wait([
+        {
+            event: "incoming_call",
+            call_id: m.collect("call_id"),
+            msg: sip_msg({
+                $rm: 'INVITE',
+                $fU: 'alice',
+                $fd: 'test.com',
+                $tU: 'bob',
+                '$hdr(X-MyHeader1)': 'abc',
+                'hdr_x_myheader2': 'def',
+            }),
+        },
+        {
+            event: 'response',
+            call_id: oc.id,
+            method: 'INVITE',
+            msg: sip_msg({
+                $rs: '100',
+                $rr: 'Trying',
+                '$(hdrcnt(via))': 1,
+                'hdr_call_id': m.collect('sip_call_id'),
+                $fU: 'alice',
+                $fd: 'test.com',
+                $tU: 'bob',
+                '$hdr(l)': '0',
+            }),
+        },
+    ], 1000)
+
+    ic = {
+        id: z.store.call_id,
+        sip_call_id: z.store.sip_call_id,
+    }
+
+    sip.call.respond(ic.id, {
+        code: 200,
+        reason:'OK',
+        headers: {
+            'X-MyHeader3': 'ghi',
+            'X-MyHeader4': 'jkl',
+        },
+    })
+
+    await z.wait([
+        {
+            event: 'media_update',
+            call_id: oc.id,
+            status: 'ok',
+        },
+        {
+            event: 'media_update',
+            call_id: ic.id,
+            status: 'ok',
+        },
+        {
+            event: 'response',
+            call_id: oc.id,
+            method: 'INVITE',
+            msg: sip_msg({
+                $rs: '200',
+                $rr: 'OK',
+                '$(hdrcnt(v))': 1,
+                $fU: 'alice',
+                $fd: 'test.com',
+                $tU: 'bob',
+                '$hdr(content-type)': 'application/sdp',
+                $rb: '!{_}a=sendrecv',
+                '$hdr(X-MyHeader3)': 'ghi',
+                '$hdr(X-MyHeader4)': 'jkl',
+            }),
+        },
+    ], 1000)
+
+    await z.sleep(200)
+
+    sip.call.start_record_wav(oc.id, {file: './oc.wav'})
+    sip.call.start_record_wav(ic.id, {file: './ic.wav'})
+
+    sip.call.start_speech_synth(oc.id, {server_url: 'ws://0.0.0.0:8080', engine: 'gss', voice: 'en-US-Standard-E', language: 'en-US', text: 'hello world', times: 1})
+    sip.call.start_speech_synth(ic.id, {server_url: 'ws://0.0.0.0:8080', engine: 'gss', voice: 'en-US-Standard-F', language: 'en-US', text: '<speak>Good morning<break time="2s"/>Good Afternoon</speak>', times: 1})
+
+    sip.call.start_speech_recog(oc.id, {server_url: 'ws://0.0.0.0:8080', engine: 'gsr', language: 'en-US'})
+    sip.call.start_speech_recog(ic.id, {server_url: 'ws://0.0.0.0:8080', engine: 'gsr', language: 'en-US'})
+
+    await z.wait([
+        {
+            event: 'speech_synth_complete',
+            call_id: ic.id,
+        },
+        {
+            event: 'speech_synth_complete',
+            call_id: oc.id,
+        },
+        {
+            event: 'speech',
+            call_id: oc.id,
+            transcript: 'good morning',
+        },
+        {
+            event: 'speech',
+            call_id: ic.id,
+            transcript: 'hello world',
+        },
+    ], 4000)
+
+    await z.wait([
+        {
+            event: 'speech',
+            call_id: oc.id,
+            transcript: ' good afternoon',
+        },
+    ], 4000)
+
+    await z.sleep(1000)
+
+    sip.call.stop_record_wav(oc.id)
+    sip.call.stop_record_wav(ic.id)
+
+    sip.call.terminate(oc.id)
+
+    await z.wait([
+        {
+            event: 'call_ended',
+            call_id: oc.id,
+        },
+        {
+            event: 'call_ended',
+            call_id: ic.id,
+        },
+        {
+            event: 'response',
+            call_id: oc.id,
+            method: 'BYE',
+            msg: sip_msg({
+                $rs: '200',
+                $rr: 'OK',
+            }),
+        },
+    ], 1000)
+
+    await z.sleep(100)
+
+    console.log("Success")
+
+    sip.stop()
+}
+
+
+test()
+.catch(e => {
+    console.error(e)
+    process.exit(1)
+})
+

package/src/event_templates.cpp

@@ -105,16 +105,16 @@ int make_evt_end_of_file(char *dest, int size, long call_id) {
       "{\"event\": \"end_of_file\", \"call_id\": %ld}", call_id);
 }
 
-int make_evt_end_of_speech(char *dest, int size, long call_id) {
+int make_evt_speech_synth_complete(char *dest, int size, long call_id) {
   return snprintf(
       dest, size,
-      "{\"event\": \"end_of_speech\", \"call_id\": %ld}", call_id);
+      "{\"event\": \"speech_synth_complete\", \"call_id\": %ld}", call_id);
 }
 
-int make_evt_speech_transcript(char *dest, int size, long call_id, char* transcript) {
+int make_evt_speech(char *dest, int size, long call_id, char* transcript) {
   return snprintf(
       dest, size,
-      "{\"event\": \"speech_transcript\", \"call_id\": %ld, \"transcript\": \"%s\"}", call_id, transcript);
+      "{\"event\": \"speech\", \"call_id\": %ld, \"transcript\": \"%s\"}", call_id, transcript);
 }
 
 int make_evt_tcp_msg(char *dest, int size, long call_id, const char *protocol, char *data, int data_len) {
@@ -122,3 +122,10 @@ int make_evt_tcp_msg(char *dest, int size, long call_id, const char *protocol, c
       dest, size,
       "{\"event\": \"%s_msg\", \"call_id\": %ld}\n%.*s", protocol, call_id, data_len, data);
 }
+
+int make_evt_ws_speech_event(char *dest, int size, long call_id, char *data, int data_len) {
+  return snprintf(
+      dest, size,
+      "{\"event\": \"ws_speech_event\", \"call_id\": %ld, \"data\": %.*s}", call_id, data_len, data);
+}
+

package/src/event_templates.hpp

@@ -36,10 +36,12 @@ int make_evt_fax_result(char *dest, int size, long call_id, int result);
 
 int make_evt_end_of_file(char *dest, int size, long call_id);
 
-int make_evt_end_of_speech(char *dest, int size, long call_id);
+int make_evt_speech_synth_complete(char *dest, int size, long call_id);
 
-int make_evt_speech_transcript(char *dest, int size, long call_id, char* transcript);
+int make_evt_speech(char *dest, int size, long call_id, char* transcript);
 
 int make_evt_tcp_msg(char *dest, int size, long call_id, const char *protocol, char *data, int data_len);
 
+int make_evt_ws_speech_event(char *dest, int size, long call_id, char *data, int data_len);
+
 #endif

package/src/pjmedia/include/pjmedia/flite_port.h

@@ -5,7 +5,7 @@
 
 PJ_BEGIN_DECL
 
-enum pjmedia_filte_option
+enum pjmedia_flite_option
 {
   PJMEDIA_SPEECH_NO_LOOP = 1
 };
@@ -25,7 +25,7 @@ PJ_DEF(pj_status_t) pjmedia_flite_port_set_eof_cb(pjmedia_port *port,
 
 PJ_DEF(pj_status_t) pjmedia_flite_port_speak( pjmedia_port *port,
                                           const char *text,
-                                          unsigned options);
+                                          int times);
 
 PJ_END_DECL