speechflow 2.0.4 → 2.1.1

package/speechflow-cli/src/speechflow-node-t2t-sentence.ts

@@ -14,13 +14,13 @@ import { Duration }       from "luxon"
 import SpeechFlowNode, { SpeechFlowChunk } from "./speechflow-node"
 import * as util                           from "./speechflow-util"
 
-/*  text stream queue element */
+/*  text stream queue element  */
 type TextQueueElement = {
-    type:         "text-frame",
-    chunk:        SpeechFlowChunk,
-    complete?:    boolean
+    type:      "text-frame",
+    chunk:     SpeechFlowChunk,
+    complete:  boolean
 } | {
-    type:         "text-eof"
+    type:      "text-eof"
 }
 
 /*  SpeechFlow node for sentence splitting  */
@@ -30,10 +30,10 @@ export default class SpeechFlowNodeT2TSentence extends SpeechFlowNode {
 
     /*  internal state  */
     private queue      = new util.Queue<TextQueueElement>()
-    private queueRecv  = this.queue.pointerUse("recv")
-    private queueSplit = this.queue.pointerUse("split")
     private queueSend  = this.queue.pointerUse("send")
-    private closing  = false
+    private queueSplit = this.queue.pointerUse("split")
+    private queueRecv  = this.queue.pointerUse("recv")
+    private closing    = false
     private workingOffTimer: ReturnType<typeof setTimeout> | null = null
 
     /*  construct node  */
@@ -41,13 +41,24 @@ export default class SpeechFlowNodeT2TSentence extends SpeechFlowNode {
         super(id, cfg, opts, args)
 
         /*  declare node configuration parameters  */
-        this.configure({})
+        this.configure({
+            timeout: { type: "number",  pos: 0, val: 3 * 1000 },
+            interim: { type: "boolean", pos: 1, val: false    }
+        })
 
         /*  declare node input/output format  */
         this.input  = "text"
         this.output = "text"
     }
 
+    /*  concatenate two payloads with proper whitespacing  */
+    private concatPayload (s1: string, s2: string) {
+        if (!(s1.match(/\s+$/) || s2.match(/^\s+/)))
+            return `${s1} ${s2}`
+        else
+            return `${s1}${s2}`
+    }
+
     /*  open node  */
     async open () {
         /*  clear destruction flag  */
@@ -78,70 +89,102 @@ export default class SpeechFlowNodeT2TSentence extends SpeechFlowNode {
                     this.queueSplit.walk(+1)
                     break
                 }
-                const chunk = element.chunk
-                const payload = chunk.payload as string
-                const m = payload.match(/^((?:.|\r?\n)+?[.;?!])\s*((?:.|\r?\n)*)$/)
-                if (m !== null) {
-                    /*  contains a sentence  */
-                    const [ , sentence, rest ] = m
-                    if (rest !== "") {
-                        /*  contains more than a sentence  */
-                        const chunk2 = chunk.clone()
-                        const duration = Duration.fromMillis(
-                            chunk.timestampEnd.minus(chunk.timestampStart).toMillis() *
-                            (sentence.length / payload.length))
-                        chunk2.timestampStart = chunk.timestampStart.plus(duration)
-                        chunk.timestampEnd    = chunk2.timestampStart
-                        chunk.payload  = sentence
-                        chunk2.payload = rest
-                        element.complete = true
-                        this.queueSplit.touch()
-                        this.queueSplit.walk(+1)
-                        this.queueSplit.insert({ type: "text-frame", chunk: chunk2 })
-                    }
-                    else {
-                        /*  contains just the sentence  */
-                        element.complete = true
-                        this.queueSplit.touch()
-                        this.queueSplit.walk(+1)
-                    }
+
+                /*  skip elements already completed  */
+                if (element.type === "text-frame" && element.chunk.kind === "final" && element.complete === true) {
+                    this.queueSplit.walk(+1)
+                    continue
                 }
-                else {
-                    /*  contains less than a sentence  */
-                    const position = this.queueSplit.position()
-                    if (position < this.queueSplit.maxPosition() - 1) {
-                        /*  merge into following chunk  */
-                        const element2 = this.queueSplit.peek(position + 1)
-                        if (element2 === undefined)
-                            break
-                        if (element2.type === "text-eof") {
+
+                /*  perform sentence splitting on input chunk  */
+                if (element.chunk.kind === "final") {
+                    const chunk = element.chunk
+                    const payload = chunk.payload as string
+                    const m = payload.match(/^((?:.|\r?\n)+?[.;?!])(?:\s+((?:.|\r?\n)+)|\s*)$/)
+                    if (m !== null) {
+                        /*  contains a sentence  */
+                        const [ , sentence, rest ] = m
+                        if (rest !== undefined && rest !== "") {
+                            /*  contains more than a sentence  */
+                            const chunk2 = chunk.clone()
+                            const duration = Duration.fromMillis(
+                                chunk.timestampEnd.minus(chunk.timestampStart).toMillis() *
+                                (sentence.length / payload.length))
+                            chunk2.timestampStart = chunk.timestampStart.plus(duration)
+                            chunk.timestampEnd    = chunk2.timestampStart
+                            chunk.payload  = sentence
+                            chunk2.payload = rest
                             element.complete = true
+                            this.queue.silent(true)
                             this.queueSplit.touch()
+                            this.queue.silent(false)
+                            this.queueSplit.walk(+1)
+                            this.queueSplit.insert({ type: "text-frame", chunk: chunk2, complete: false })
+                        }
+                        else {
+                            /*  contains just the sentence  */
+                            element.complete = true
+                            this.queue.silent(true)
+                            this.queueSplit.silent(true)
+                            const position = this.queueSplit.position()
                             this.queueSplit.walk(+1)
+                            this.queue.silent(false)
+                            this.queueSplit.silent(false)
+                            this.queueSplit.touch(position)
+                        }
+                    }
+                    else {
+                        /*  contains less than a sentence  */
+                        const position = this.queueSplit.position()
+                        if (position < this.queueSplit.maxPosition() - 1) {
+                            /*  merge into following chunk  */
+                            const element2 = this.queueSplit.peek(position + 1)
+                            if (element2 === undefined)
+                                break
+                            if (element2.type === "text-eof") {
+                                /*  no more chunks: output as final
+                                    (perhaps incomplete sentence at end of stream)  */
+                                element.complete = true
+                                this.queueSplit.walk(+1)
+                                this.queueSplit.touch(this.queueSplit.position() - 1)
+                                break
+                            }
+                            if (element2.chunk.kind === "final") {
+                                /*  merge into following chunk  */
+                                element2.chunk.timestampStart = element.chunk.timestampStart
+                                element2.chunk.payload = this.concatPayload(element.chunk.payload as string,
+                                    element2.chunk.payload as string)
+
+                                /*  remove current element and touch now current element  */
+                                this.queue.silent(true)
+                                this.queueSplit.delete()
+                                this.queue.silent(false)
+                                this.queueSplit.touch()
+                            }
+                            else
+                                break
+                        }
+                        else {
+                            /*  no following chunk yet  */
                             break
                         }
-                        element2.chunk.timestampStart = element.chunk.timestampStart
-                        element2.chunk.payload =
-                            (element.chunk.payload  as string) + " " +
-                            (element2.chunk.payload as string)
-                        this.queueSplit.delete()
-                        this.queueSplit.touch()
                     }
-                    else
-                        break
                 }
+                else
+                    break
             }
 
             /*  re-initiate working off round (if still not destroyed)  */
-            workingOff = false
             if (!this.closing) {
                 this.workingOffTimer = setTimeout(workOffQueue, 100)
                 this.queue.once("write", workOffQueue)
             }
+            workingOff = false
         }
         this.queue.once("write", workOffQueue)
 
         /*  provide Duplex stream and internally attach to classifier  */
+        let previewed = false
         const self = this
         this.stream = new Stream.Duplex({
             writableObjectMode: true,
@@ -150,7 +193,7 @@ export default class SpeechFlowNodeT2TSentence extends SpeechFlowNode {
             highWaterMark:      1,
 
             /*  receive text chunk (writable side of stream)  */
-            write (chunk: SpeechFlowChunk, encoding, callback) {
+            write (chunk: SpeechFlowChunk, encoding: BufferEncoding, callback: (error?: Error | null) => void) {
                 if (self.closing)
                     callback(new Error("stream already destroyed"))
                 else if (Buffer.isBuffer(chunk.payload))
@@ -158,14 +201,30 @@ export default class SpeechFlowNodeT2TSentence extends SpeechFlowNode {
                 else if (chunk.payload.length === 0)
                     callback()
                 else {
-                    self.log("info", `received text: ${JSON.stringify(chunk.payload)}`)
-                    self.queueRecv.append({ type: "text-frame", chunk })
+                    /*  final chunks: queue for sentence splitting  */
+                    self.log("info", `received text (${chunk.kind}): ${JSON.stringify(chunk.payload)}`)
+                    const recvPos = self.queueRecv.position()
+                    if (recvPos > 0) {
+                        const element = self.queueRecv.peek(recvPos - 1)
+                        if (element) {
+                            if (element.type === "text-eof") {
+                                callback(new Error("received text input after end-of-stream"))
+                                return
+                            }
+                            if (element.chunk.kind === "intermediate") {
+                                self.queueRecv.walk(-1)
+                                self.queueRecv.delete()
+                            }
+                        }
+                    }
+                    previewed = false
+                    self.queueRecv.append({ type: "text-frame", chunk, complete: false })
                     callback()
                 }
             },
 
             /*  receive no more text chunks (writable side of stream)  */
-            final (callback) {
+            final (callback: (error?: Error | null) => void) {
                 if (self.closing) {
                     callback()
                     return
@@ -192,6 +251,8 @@ export default class SpeechFlowNodeT2TSentence extends SpeechFlowNode {
                     else if (element !== undefined
                         && element.type === "text-frame"
                         && element.complete === true) {
+                        /*  send all consecutive complete chunks  */
+                        let eofSeen = false
                         while (true) {
                             const nextElement = self.queueSend.peek()
                             if (nextElement === undefined)
@@ -199,16 +260,47 @@ export default class SpeechFlowNodeT2TSentence extends SpeechFlowNode {
                             else if (nextElement.type === "text-eof") {
                                 this.push(null)
                                 self.queueSend.walk(+1)
+                                eofSeen = true
                                 break
                             }
                             else if (nextElement.type === "text-frame"
                                 && nextElement.complete !== true)
                                 break
-                            self.log("info", `send text: ${JSON.stringify(nextElement.chunk.payload)}`)
+                            self.log("info", `send text 1 (${nextElement.chunk.kind}): ${JSON.stringify(nextElement.chunk.payload)} pos=${self.queueSend.position()}`)
                             this.push(nextElement.chunk)
                             self.queueSend.walk(+1)
                             self.queue.trim()
                         }
+
+                        /*  wait for more data (unless end-of-stream was reached)  */
+                        if (!eofSeen && !self.closing)
+                            self.queue.once("write", flushPendingChunks)
+                    }
+                    else if (element !== undefined
+                        && element.type === "text-frame"
+                        && element.complete === false
+                        && !previewed
+                        && self.params.interim === true) {
+                        /*  merge together all still queued elements and
+                            send this out as an intermediate chunk as preview  */
+                        const previewChunk = element.chunk.clone()
+                        previewChunk.kind = "intermediate"
+                        for (let pos = self.queueSend.position() + 1; pos < self.queueSend.maxPosition(); pos++) {
+                            const element2 = self.queueSend.peek(pos)
+                            if (!element2)
+                                continue
+                            if (element2.type === "text-eof")
+                                break
+                            previewChunk.payload = self.concatPayload(
+                                previewChunk.payload as string, element2.chunk.payload as string)
+                        }
+                        this.push(previewChunk)
+                        self.log("info", `send text 2 (intermediate): ${JSON.stringify(previewChunk.payload)}`)
+                        previewed = true
+
+                        /*  wait for more data  */
+                        if (!self.closing)
+                            self.queue.once("write", flushPendingChunks)
                     }
                     else if (!self.closing)
                         self.queue.once("write", flushPendingChunks)
@@ -223,7 +315,7 @@ export default class SpeechFlowNodeT2TSentence extends SpeechFlowNode {
         /*  indicate closing  */
         this.closing = true
 
-        /*  clean up timer  */
+        /*  clean up timers  */
         if (this.workingOffTimer !== null) {
             clearTimeout(this.workingOffTimer)
             this.workingOffTimer = null

package/speechflow-cli/src/speechflow-util-queue.ts

@@ -97,6 +97,7 @@ export type QueueElement = { type: string }
 export class QueuePointer<T extends QueueElement> extends EventEmitter {
     /*  internal state  */
     private index = 0
+    private silence = false
 
     /*  construction  */
     constructor (
@@ -107,6 +108,17 @@ export class QueuePointer<T extends QueueElement> extends EventEmitter {
         this.setMaxListeners(100)
     }
 
+    /*  control silence operation  */
+    silent (silence: boolean) {
+        this.silence = silence
+    }
+
+    /*  notify about operation  */
+    notify (event: string, info: any) {
+        if (!this.silence)
+            this.emit(event, info)
+    }
+
     /*  positioning operations  */
     maxPosition () {
         return this.queue.elements.length
@@ -114,7 +126,7 @@ export class QueuePointer<T extends QueueElement> extends EventEmitter {
     position (index?: number): number {
         if (index !== undefined) {
             this.index = Math.max(0, Math.min(index, this.queue.elements.length))
-            this.emit("position", this.index)
+            this.notify("position", this.index)
         }
         return this.index
     }
@@ -125,19 +137,19 @@ export class QueuePointer<T extends QueueElement> extends EventEmitter {
         else if (num < 0)
             this.index = Math.max(this.index + num, 0)
         if (this.index !== indexOld)
-            this.emit("position", { start: this.index })
+            this.notify("position", { start: this.index })
     }
     walkForwardUntil (type: T["type"]) {
         while (this.index < this.queue.elements.length
             && this.queue.elements[this.index].type !== type)
             this.index++
-        this.emit("position", { start: this.index })
+        this.notify("position", { start: this.index })
     }
     walkBackwardUntil (type: T["type"]) {
         while (this.index > 0
             && this.queue.elements[this.index].type !== type)
             this.index--
-        this.emit("position", { start: this.index })
+        this.notify("position", { start: this.index })
     }
 
     /*  search operations  */
@@ -146,7 +158,7 @@ export class QueuePointer<T extends QueueElement> extends EventEmitter {
         while (position < this.queue.elements.length
             && this.queue.elements[position].type !== type)
             position++
-        this.emit("search", { start: this.index, end: position })
+        this.notify("search", { start: this.index, end: position })
         return position
     }
     searchBackward (type: T["type"]) {
@@ -154,24 +166,24 @@ export class QueuePointer<T extends QueueElement> extends EventEmitter {
         while (position > 0
             && this.queue.elements[position].type !== type)
             position--
-        this.emit("search", { start: position, end: this.index })
+        this.notify("search", { start: position, end: this.index })
         return position
     }
 
     /*  reading operations  */
-    peek (position?: number) {
+    peek (position?: number): T | undefined {
         if (position === undefined)
             position = this.index
         position = Math.max(0, Math.min(position, this.queue.elements.length))
         const element = this.queue.elements[position]
-        this.queue.emit("read", { start: position, end: position })
+        this.queue.notify("read", { start: position, end: position })
         return element
     }
-    read () {
+    read (): T | undefined {
         const element = this.queue.elements[this.index]
         if (this.index < this.queue.elements.length)
             this.index++
-        this.queue.emit("read", { start: this.index - 1, end: this.index - 1 })
+        this.queue.notify("read", { start: this.index - 1, end: this.index - 1 })
         return element
     }
     slice (size?: number) {
@@ -186,30 +198,32 @@ export class QueuePointer<T extends QueueElement> extends EventEmitter {
             slice = this.queue.elements.slice(this.index)
             this.index = this.queue.elements.length
         }
-        this.queue.emit("read", { start, end: this.index })
+        this.queue.notify("read", { start, end: this.index })
         return slice
     }
 
     /*  writing operations  */
-    touch () {
-        if (this.index >= this.queue.elements.length)
-            throw new Error("cannot touch after last element")
-        this.queue.emit("write", { start: this.index, end: this.index + 1 })
+    touch (position?: number) {
+        if (position === undefined)
+            position = this.index
+        if (position >= this.queue.elements.length)
+            throw new Error(`cannot touch after last element ${position} ${this.queue.elements.length}`)
+        this.queue.notify("write", { start: position, end: position, op: "touch" })
     }
     append (element: T) {
         this.queue.elements.push(element)
         this.index = this.queue.elements.length
-        this.queue.emit("write", { start: this.index - 1, end: this.index - 1 })
+        this.queue.notify("write", { start: this.index - 1, end: this.index - 1, op: "append" })
     }
     insert (element: T) {
         this.queue.elements.splice(this.index, 0, element)
-        this.queue.emit("write", { start: this.index - 1, end: this.index })
+        this.queue.notify("write", { start: this.index, end: this.index, op: "insert" })
     }
     delete () {
         if (this.index >= this.queue.elements.length)
             throw new Error("cannot delete after last element")
         this.queue.elements.splice(this.index, 1)
-        this.queue.emit("write", { start: this.index, end: this.index })
+        this.queue.notify("write", { start: this.index, end: this.index, op: "delete" })
     }
 }
 
@@ -217,10 +231,18 @@ export class QueuePointer<T extends QueueElement> extends EventEmitter {
 export class Queue<T extends QueueElement> extends EventEmitter {
     public elements: T[] = []
     private pointers = new Map<string, QueuePointer<T>>()
+    private silence = false
     constructor () {
         super()
         this.setMaxListeners(100)
     }
+    silent (silence: boolean) {
+        this.silence = silence
+    }
+    notify (event: string, info: any) {
+        if (!this.silence)
+            this.emit(event, info)
+    }
     pointerUse (name: string): QueuePointer<T> {
         if (!this.pointers.has(name))
             this.pointers.set(name, new QueuePointer<T>(name, this))
@@ -234,9 +256,10 @@ export class Queue<T extends QueueElement> extends EventEmitter {
     trim (): void {
         /*  determine minimum pointer position  */
         let min = this.elements.length
-        for (const pointer of this.pointers.values())
+        for (const pointer of this.pointers.values()) {
             if (min > pointer.position())
                 min = pointer.position()
+        }
 
         /*  trim the maximum amount of first elements  */
         if (min > 0) {
@@ -245,6 +268,8 @@ export class Queue<T extends QueueElement> extends EventEmitter {
             /*  shift all pointers  */
             for (const pointer of this.pointers.values())
                 pointer.position(pointer.position() - min)
+
+            this.notify("write", { start: 0, end: min, op: "trim" })
         }
     }
 }