sip-lab 1.23.0 → 1.24.0

package/src/pjmedia/src/pjmedia/pocketsphinx_port.c

@@ -18,7 +18,7 @@
  * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA 
  */
 
-#include <pjmedia/pocketsphinx.h>
+#include <pocketsphinx_port.h>
 #include <pjmedia/errno.h>
 #include <pjmedia/port.h>
 #include <pj/assert.h>
@@ -44,43 +44,31 @@ struct pocketsphinx_t
 {
     struct pjmedia_port	base;
 
-    pj_bool_t        subscribed;
-    void           (*cb)(pjmedia_port*, void*, char*);
-};
-
-/*
- * Register a callback to be called when we get translation
- */
-PJ_DEF(pj_status_t) pjmedia_pocketsphinx_port_set_speech_cb(pjmedia_port *port,
-                               void *user_data,
-                               void (*cb)(pjmedia_port *port,
-                                          void *usr_data, char *transcript))
-{
-    struct pocketsphinx_t *flite;
+    ps_decoder_t *decoder;
+    ps_config_t *config;
+    ps_endpointer_t *ep;
 
-    /* Sanity check */
-    PJ_ASSERT_RETURN(port, -PJ_EINVAL);
+    unsigned in_spf;
+    unsigned out_spf;
 
-    /* Check that this is really a pocketsphinx port */
-    PJ_ASSERT_RETURN(port->info.signature == SIGNATURE, -PJ_EINVALIDOP);
+    short *samples;
+    unsigned sample_count;
 
-    pocketsphinx = (struct pocketsphinx_t*) port;
-
-    pocketsphinx->base.port_data.pdata = user_data;
-    pocketsphinx->cb = cb;
-
-    return PJ_SUCCESS;
-}
+    pj_bool_t subscribed;
+    void (*cb)(pjmedia_port*, void*, char*);
+    void *cb_user_data;
 
+    char transcript[4096];
+};
 
 static pj_status_t speech_on_event(pjmedia_event *event,
                                  void *user_data)
 {
-    struct pocketsphinx_t *pocketsphinx = (struct pocketsphinx_t*)user_data;
+    struct pocketsphinx_t *port = (struct pocketsphinx_t*)user_data;
 
     if (event->type == PJMEDIA_EVENT_CALLBACK) {
-        if (pocketsphinx->cb)
-            (*pocketsphinx->cb)(&flite->base, flite->base.port_data.pdata, "transcript");
+        if (port->cb)
+            (*port->cb)(&port->base, port->cb_user_data, port->transcript);
     }
     
     return PJ_SUCCESS;
@@ -91,52 +79,176 @@ PJ_DEF(pj_status_t) pjmedia_pocketsphinx_port_create( pj_pool_t *pool,
 				unsigned channel_count,
 				unsigned samples_per_frame,
 				unsigned bits_per_sample,
+                void (*cb)(pjmedia_port*, void *user_data, char *transcript),
+                void *cb_user_data,
 				pjmedia_port **p_port)
 {
-    struct pocketsphinx *pocketsphinx;
+    struct pocketsphinx_t *port;
     const pj_str_t name = pj_str("pocketsphinx");
 
-    PJ_ASSERT_RETURN(pool && clock_rate && channel_count && 
+    PJ_ASSERT_RETURN(pool && clock_rate && channel_count == 1 && 
 		     samples_per_frame && bits_per_sample == 16 && 
 		     p_port != NULL, PJ_EINVAL);
 
     PJ_ASSERT_RETURN(pool && p_port, PJ_EINVAL);
 
-    pocketsphinx = PJ_POOL_ZALLOC_T(pool, struct pocketsphinx);
+    port = PJ_POOL_ZALLOC_T(pool, struct pocketsphinx_t);
     PJ_ASSERT_RETURN(pool != NULL, PJ_ENOMEM);
 
-    pjmedia_port_info_init(&pocketsphinx->base.info, &name, SIGNATURE, clock_rate,
+    pjmedia_port_info_init(&port->base.info, &name, SIGNATURE, clock_rate,
 			   channel_count, bits_per_sample, samples_per_frame);
 
-    pocketsphinx->base.put_frame = &pocketsphinx_put_frame;
-    pocketsphinx->base.on_destroy = &pocketsphinx_on_destroy;
+    port->base.put_frame = &pocketsphinx_put_frame;
+    port->base.on_destroy = &pocketsphinx_on_destroy;
 
-    pocketsphinx->dtmf_cb = cb;
-    pocketsphinx->dtmf_cb_user_data = user_data;
+    port->cb = cb;
+    port->cb_user_data = cb_user_data;
 
-    dtmf_rx_init(&pocketsphinx->state, NULL, NULL);
-    dtmf_rx_set_realtime_callback(&pocketsphinx->state,
-					&pocketsphinx_digit_callback,
-					(void*)pocketsphinx);	 
+    port->config = ps_config_init(NULL);
+    ps_default_search_args(port->config);
+    
+    if ((port->decoder = ps_init(port->config)) == NULL) {
+        TRACE_((THIS_FILE, "pocketsphinx port: decoder init failed\n"));
+        return !PJ_SUCCESS;
+    }
+
+    if ((port->ep = ps_endpointer_init(0, 0.0, 0, clock_rate, 0)) == NULL) {
+        TRACE_((THIS_FILE, "pocketsphinx port: endpointer init failed\n"));
+        return !PJ_SUCCESS;
+    }
+
+    port->in_spf = samples_per_frame;
+    port->out_spf = ps_endpointer_frame_size(port->ep);
 
-    TRACE_((THIS_FILE, "pocketsphinx created: %u/%u/%u/%u", clock_rate, 
+    port->samples = (short*) pj_pool_alloc(pool, port->out_spf * sizeof(short));
+    if (port->samples == NULL) {
+        TRACE_(("Failed to allocate buffer for samples\n"));
+        return !PJ_SUCCESS;
+    }
+
+    TRACE_((THIS_FILE, "pocketsphinx port created: %u/%u/%u/%u", clock_rate, 
 	    channel_count, samples_per_frame, bits_per_sample));
 
-    *p_port = &pocketsphinx->base; 
+    printf("pocketsphinx_create in_spf=%i out_spf=%i\n", port->in_spf, port->out_spf);
+    *p_port = &port->base; 
     return PJ_SUCCESS;
 }
 
+void feed(struct pocketsphinx_t *port, short *frame) {
+    const int16 *speech;
+    int prev_in_speech = ps_endpointer_in_speech(port->ep);
+    speech = ps_endpointer_process(port->ep, frame);
+    if (speech != NULL) {
+        const char *hyp;
+        if (!prev_in_speech) {
+            printf("pocketsphinx speech start at %.2f\n", ps_endpointer_speech_start(port->ep));
+            ps_start_utt(port->decoder);
+        }
+        if (ps_process_raw(port->decoder, speech, port->out_spf, FALSE, FALSE) < 0) {
+            printf("pocketsphinx ps_process_raw() failed\n");
+            return;
+        }
+        if ((hyp = ps_get_hyp(port->decoder, NULL)) != NULL) {
+            //printf("pocketsphinx partial result: %s\n", hyp);
+        }
+        if (!ps_endpointer_in_speech(port->ep)) {
+            printf("Speech end at %.2f\n", ps_endpointer_speech_end(port->ep));
+            ps_end_utt(port->decoder);
+            if ((hyp = ps_get_hyp(port->decoder, NULL)) != NULL) {
+                printf("pocketsphinx speech: %s\n", hyp);
+                if(strlen(hyp) == 0) return;
+
+                strncpy(port->transcript, hyp, sizeof(port->transcript) - 1);
+
+                // Ensure the destination string is null-terminated
+                port->transcript[sizeof(port->transcript) - 1] = '\0';
+
+                if(port->cb) {
+                    if (!port->subscribed) {
+                        pj_status_t status = pjmedia_event_subscribe(NULL, &speech_on_event,
+                                                         port, port);
+                        port->subscribed = (status == PJ_SUCCESS)? PJ_TRUE:
+                                            PJ_FALSE;
+                    }
+
+                    if (port->subscribed) {
+                        pjmedia_event event;
+
+                        pjmedia_event_init(&event, PJMEDIA_EVENT_CALLBACK,
+                                           NULL, port);
+                        pjmedia_event_publish(NULL, port, &event,
+                                              PJMEDIA_EVENT_PUBLISH_POST_EVENT);
+                    }
+                }
+            }
+        }
+    }
+}
+
+unsigned feed_one(struct pocketsphinx_t *port, pjmedia_frame *frame){
+    unsigned used_samples = port->out_spf - port->sample_count;
+    memcpy((short*)port->samples + port->sample_count, frame->buf, used_samples * sizeof(short));
+    feed(port, port->samples);
+    port->sample_count = 0;
+    return used_samples;
+}
+
+void feed_all(struct pocketsphinx_t *port, pjmedia_frame *frame) {
+    unsigned samples = frame->size / 2;
+    unsigned used_samples = 0;
+   if(port->sample_count > 0) {
+       used_samples = feed_one(port, frame); 
+    }
+
+    short *out_frame = frame->buf + used_samples;
+    samples -= used_samples;
+    unsigned count = 0;
+    while(samples >= port->out_spf) {
+        feed(port, out_frame);
+        count++;
+        out_frame += (count * port->out_spf);
+        samples -= port->out_spf;
+    }
+
+    if(samples) {
+        memcpy(port->samples, out_frame, samples * sizeof(short));
+        port->sample_count = samples;
+    }
+}
+
 static pj_status_t pocketsphinx_put_frame(pjmedia_port *this_port, 
 				  pjmedia_frame *frame)
 {
     if(frame->type != PJMEDIA_FRAME_TYPE_AUDIO) return PJ_SUCCESS;
 
-    struct pocketsphinx *dport = (struct pocketsphinx*) this_port;
-    dtmf_rx(&dport->state, (const pj_int16_t*)frame->buf,
-		PJMEDIA_PIA_SPF(&dport->base.info));
+    struct pocketsphinx_t *port = (struct pocketsphinx_t*) this_port;
 
-    return PJ_SUCCESS;
+    if(port->in_spf == port->out_spf) {
+        //printf("feed\n");
+        feed(port, (short*)frame->buf);
+        return;
+    }
 
+    if(port->in_spf > port->out_spf) {
+        //printf("feed_all\n");
+        feed_all(port, frame);
+        return;
+    }
+
+    unsigned samples = frame->size / 2;
+    if(samples + port->sample_count >= port->out_spf) {
+        // enough to feed once
+        //printf("feed_one\n");
+        feed_one(port, frame);
+        return;
+    }
+            
+    // not enough to feed. 
+    //printf("not enough to feed\n");
+    memcpy((short*)port->samples + port->sample_count, frame->buf, samples * sizeof(short));
+    port->sample_count += samples;
+
+    return PJ_SUCCESS;
 }
 
 /*
@@ -144,6 +256,17 @@ static pj_status_t pocketsphinx_put_frame(pjmedia_port *this_port,
  */
 static pj_status_t pocketsphinx_on_destroy(pjmedia_port *this_port)
 {
+    struct pocketsphinx_t *port = (struct pocketsphinx_t*) this_port;
+
+    ps_endpointer_free(port->ep);
+    ps_free(port->decoder);
+    ps_config_free(port->config);
+ 
+    if (port->subscribed) {
+        pjmedia_event_unsubscribe(NULL, &speech_on_event, port, port);
+        port->subscribed = PJ_FALSE;
+    }
+
     return PJ_SUCCESS;
 }