speechflow 2.1.0 → 2.1.1

package/speechflow-cli/src/speechflow-node-t2t-sentence.ts

@@ -18,8 +18,7 @@ import * as util                           from "./speechflow-util"
 type TextQueueElement = {
     type:      "text-frame",
     chunk:     SpeechFlowChunk,
-    preview?:  "pending" | "sent",
-    complete?: boolean
+    complete:  boolean
 } | {
     type:      "text-eof"
 }
@@ -31,12 +30,11 @@ export default class SpeechFlowNodeT2TSentence extends SpeechFlowNode {
 
     /*  internal state  */
     private queue      = new util.Queue<TextQueueElement>()
-    private queueRecv  = this.queue.pointerUse("recv")
-    private queueSplit = this.queue.pointerUse("split")
     private queueSend  = this.queue.pointerUse("send")
-    private closing  = false
+    private queueSplit = this.queue.pointerUse("split")
+    private queueRecv  = this.queue.pointerUse("recv")
+    private closing    = false
     private workingOffTimer: ReturnType<typeof setTimeout> | null = null
-    private previewTimer:    ReturnType<typeof setTimeout> | null = null
 
     /*  construct node  */
     constructor (id: string, cfg: { [ id: string ]: any }, opts: { [ id: string ]: any }, args: any[]) {
@@ -44,7 +42,8 @@ export default class SpeechFlowNodeT2TSentence extends SpeechFlowNode {
 
         /*  declare node configuration parameters  */
         this.configure({
-            timeout: { type: "number", pos: 0, val: 3 * 1000 }
+            timeout: { type: "number",  pos: 0, val: 3 * 1000 },
+            interim: { type: "boolean", pos: 1, val: false    }
         })
 
         /*  declare node input/output format  */
@@ -52,6 +51,14 @@ export default class SpeechFlowNodeT2TSentence extends SpeechFlowNode {
         this.output = "text"
     }
 
+    /*  concatenate two payloads with proper whitespacing  */
+    private concatPayload (s1: string, s2: string) {
+        if (!(s1.match(/\s+$/) || s2.match(/^\s+/)))
+            return `${s1} ${s2}`
+        else
+            return `${s1}${s2}`
+    }
+
     /*  open node  */
     async open () {
         /*  clear destruction flag  */
@@ -83,84 +90,101 @@ export default class SpeechFlowNodeT2TSentence extends SpeechFlowNode {
                     break
                 }
 
-                /*  perform sentence splitting on input chunk  */
-                const chunk = element.chunk
-                const payload = chunk.payload as string
-                const m = payload.match(/^((?:.|\r?\n)+?[.;?!])\s*((?:.|\r?\n)*)$/)
-                if (m !== null) {
-                    /*  contains a sentence  */
-                    const [ , sentence, rest ] = m
-                    if (rest !== "") {
-                        /*  contains more than a sentence  */
-                        const chunk2 = chunk.clone()
-                        const duration = Duration.fromMillis(
-                            chunk.timestampEnd.minus(chunk.timestampStart).toMillis() *
-                            (sentence.length / payload.length))
-                        chunk2.timestampStart = chunk.timestampStart.plus(duration)
-                        chunk.timestampEnd    = chunk2.timestampStart
-                        chunk.payload  = sentence
-                        chunk2.payload = rest
-                        element.complete = true
-                        this.queueSplit.touch()
-                        this.queueSplit.walk(+1)
-                        this.queueSplit.insert({ type: "text-frame", chunk: chunk2 })
-                    }
-                    else {
-                        /*  contains just the sentence  */
-                        element.complete = true
-                        this.queueSplit.touch()
-                        this.queueSplit.walk(+1)
-                    }
+                /*  skip elements already completed  */
+                if (element.type === "text-frame" && element.chunk.kind === "final" && element.complete === true) {
+                    this.queueSplit.walk(+1)
+                    continue
                 }
-                else {
-                    /*  contains less than a sentence  */
-                    const position = this.queueSplit.position()
-                    if (position < this.queueSplit.maxPosition() - 1) {
-                        /*  merge into following chunk  */
-                        const element2 = this.queueSplit.peek(position + 1)
-                        if (element2 === undefined)
-                            break
-                        if (element2.type === "text-eof") {
-                            /*  no more chunks: output as final
-                                (perhaps incomplete sentence at end of stream)  */
+
+                /*  perform sentence splitting on input chunk  */
+                if (element.chunk.kind === "final") {
+                    const chunk = element.chunk
+                    const payload = chunk.payload as string
+                    const m = payload.match(/^((?:.|\r?\n)+?[.;?!])(?:\s+((?:.|\r?\n)+)|\s*)$/)
+                    if (m !== null) {
+                        /*  contains a sentence  */
+                        const [ , sentence, rest ] = m
+                        if (rest !== undefined && rest !== "") {
+                            /*  contains more than a sentence  */
+                            const chunk2 = chunk.clone()
+                            const duration = Duration.fromMillis(
+                                chunk.timestampEnd.minus(chunk.timestampStart).toMillis() *
+                                (sentence.length / payload.length))
+                            chunk2.timestampStart = chunk.timestampStart.plus(duration)
+                            chunk.timestampEnd    = chunk2.timestampStart
+                            chunk.payload  = sentence
+                            chunk2.payload = rest
                             element.complete = true
+                            this.queue.silent(true)
                             this.queueSplit.touch()
+                            this.queue.silent(false)
                             this.queueSplit.walk(+1)
-                            break
+                            this.queueSplit.insert({ type: "text-frame", chunk: chunk2, complete: false })
+                        }
+                        else {
+                            /*  contains just the sentence  */
+                            element.complete = true
+                            this.queue.silent(true)
+                            this.queueSplit.silent(true)
+                            const position = this.queueSplit.position()
+                            this.queueSplit.walk(+1)
+                            this.queue.silent(false)
+                            this.queueSplit.silent(false)
+                            this.queueSplit.touch(position)
                         }
-
-                        /*  merge into following chunk  */
-                        element2.chunk.timestampStart = element.chunk.timestampStart
-                        element2.chunk.payload =
-                            (element.chunk.payload  as string) + " " +
-                            (element2.chunk.payload as string)
-
-                        /*  reset preview state (merged content needs new preview)  */
-                        element2.preview = undefined
-                        this.queueSplit.delete()
-                        this.queueSplit.touch()
                     }
                     else {
-                        /*  no following chunk yet: mark for intermediate preview output  */
-                        if (element.preview !== "sent") {
-                            element.preview = "pending"
-                            this.queueSplit.touch()
+                        /*  contains less than a sentence  */
+                        const position = this.queueSplit.position()
+                        if (position < this.queueSplit.maxPosition() - 1) {
+                            /*  merge into following chunk  */
+                            const element2 = this.queueSplit.peek(position + 1)
+                            if (element2 === undefined)
+                                break
+                            if (element2.type === "text-eof") {
+                                /*  no more chunks: output as final
+                                    (perhaps incomplete sentence at end of stream)  */
+                                element.complete = true
+                                this.queueSplit.walk(+1)
+                                this.queueSplit.touch(this.queueSplit.position() - 1)
+                                break
+                            }
+                            if (element2.chunk.kind === "final") {
+                                /*  merge into following chunk  */
+                                element2.chunk.timestampStart = element.chunk.timestampStart
+                                element2.chunk.payload = this.concatPayload(element.chunk.payload as string,
+                                    element2.chunk.payload as string)
+
+                                /*  remove current element and touch now current element  */
+                                this.queue.silent(true)
+                                this.queueSplit.delete()
+                                this.queue.silent(false)
+                                this.queueSplit.touch()
+                            }
+                            else
+                                break
+                        }
+                        else {
+                            /*  no following chunk yet  */
+                            break
                         }
-                        break
                     }
                 }
+                else
+                    break
             }
 
             /*  re-initiate working off round (if still not destroyed)  */
-            workingOff = false
             if (!this.closing) {
                 this.workingOffTimer = setTimeout(workOffQueue, 100)
                 this.queue.once("write", workOffQueue)
             }
+            workingOff = false
         }
         this.queue.once("write", workOffQueue)
 
         /*  provide Duplex stream and internally attach to classifier  */
+        let previewed = false
         const self = this
         this.stream = new Stream.Duplex({
             writableObjectMode: true,
@@ -169,37 +193,38 @@ export default class SpeechFlowNodeT2TSentence extends SpeechFlowNode {
             highWaterMark:      1,
 
             /*  receive text chunk (writable side of stream)  */
-            write (chunk: SpeechFlowChunk, encoding, callback) {
+            write (chunk: SpeechFlowChunk, encoding: BufferEncoding, callback: (error?: Error | null) => void) {
                 if (self.closing)
                     callback(new Error("stream already destroyed"))
                 else if (Buffer.isBuffer(chunk.payload))
                     callback(new Error("expected text input as string chunks"))
                 else if (chunk.payload.length === 0)
                     callback()
-                else if (chunk.kind === "intermediate") {
-                    /*  intermediate chunks: pass through immediately (bypass queue)  */
-                    self.log("info", `received text (${chunk.kind}): ${JSON.stringify(chunk.payload)}`)
-                    self.log("info", `send text (intermediate pass-through): ${JSON.stringify(chunk.payload)}`)
-                    this.push(chunk)
-                    callback()
-                }
                 else {
                     /*  final chunks: queue for sentence splitting  */
                     self.log("info", `received text (${chunk.kind}): ${JSON.stringify(chunk.payload)}`)
-
-                    /*  cancel any pending preview timeout  */
-                    if (self.previewTimer !== null) {
-                        clearTimeout(self.previewTimer)
-                        self.previewTimer = null
+                    const recvPos = self.queueRecv.position()
+                    if (recvPos > 0) {
+                        const element = self.queueRecv.peek(recvPos - 1)
+                        if (element) {
+                            if (element.type === "text-eof") {
+                                callback(new Error("received text input after end-of-stream"))
+                                return
+                            }
+                            if (element.chunk.kind === "intermediate") {
+                                self.queueRecv.walk(-1)
+                                self.queueRecv.delete()
+                            }
+                        }
                     }
-
-                    self.queueRecv.append({ type: "text-frame", chunk })
+                    previewed = false
+                    self.queueRecv.append({ type: "text-frame", chunk, complete: false })
                     callback()
                 }
             },
 
             /*  receive no more text chunks (writable side of stream)  */
-            final (callback) {
+            final (callback: (error?: Error | null) => void) {
                 if (self.closing) {
                     callback()
                     return
@@ -227,6 +252,7 @@ export default class SpeechFlowNodeT2TSentence extends SpeechFlowNode {
                         && element.type === "text-frame"
                         && element.complete === true) {
                         /*  send all consecutive complete chunks  */
+                        let eofSeen = false
                         while (true) {
                             const nextElement = self.queueSend.peek()
                             if (nextElement === undefined)
@@ -234,49 +260,43 @@ export default class SpeechFlowNodeT2TSentence extends SpeechFlowNode {
                             else if (nextElement.type === "text-eof") {
                                 this.push(null)
                                 self.queueSend.walk(+1)
+                                eofSeen = true
                                 break
                             }
                             else if (nextElement.type === "text-frame"
                                 && nextElement.complete !== true)
                                 break
-                            self.log("info", `send text (${nextElement.chunk.kind}): ${JSON.stringify(nextElement.chunk.payload)}`)
+                            self.log("info", `send text 1 (${nextElement.chunk.kind}): ${JSON.stringify(nextElement.chunk.payload)} pos=${self.queueSend.position()}`)
                             this.push(nextElement.chunk)
                             self.queueSend.walk(+1)
                             self.queue.trim()
                         }
+
+                        /*  wait for more data (unless end-of-stream was reached)  */
+                        if (!eofSeen && !self.closing)
+                            self.queue.once("write", flushPendingChunks)
                     }
                     else if (element !== undefined
                         && element.type === "text-frame"
-                        && element.preview === "pending") {
-                        /*  send intermediate preview (without advancing pointer)  */
+                        && element.complete === false
+                        && !previewed
+                        && self.params.interim === true) {
+                        /*  merge together all still queued elements and
+                            send this out as an intermediate chunk as preview  */
                         const previewChunk = element.chunk.clone()
                         previewChunk.kind = "intermediate"
-                        self.log("info", `send text (intermediate preview): ${JSON.stringify(previewChunk.payload)}`)
-                        this.push(previewChunk)
-                        element.preview = "sent"
-                        self.queueSend.touch()
-
-                        /*  start preview timeout (if configured)  */
-                        const timeout = self.params.timeout as number
-                        if (timeout > 0 && self.previewTimer === null) {
-                            self.previewTimer = setTimeout(() => {
-                                self.previewTimer = null
-                                if (self.closing)
-                                    return
-
-                                /*  promote preview to final chunk  */
-                                const el = self.queueSend.peek()
-                                if (el !== undefined
-                                    && el.type === "text-frame"
-                                    && el.preview === "sent"
-                                    && el.complete !== true) {
-                                    self.log("info", `timeout: promoting intermediate to final: ${JSON.stringify(el.chunk.payload)}`)
-                                    el.complete = true
-                                    self.queueSend.touch()
-                                    self.queue.emit("write")
-                                }
-                            }, timeout)
+                        for (let pos = self.queueSend.position() + 1; pos < self.queueSend.maxPosition(); pos++) {
+                            const element2 = self.queueSend.peek(pos)
+                            if (!element2)
+                                continue
+                            if (element2.type === "text-eof")
+                                break
+                            previewChunk.payload = self.concatPayload(
+                                previewChunk.payload as string, element2.chunk.payload as string)
                         }
+                        this.push(previewChunk)
+                        self.log("info", `send text 2 (intermediate): ${JSON.stringify(previewChunk.payload)}`)
+                        previewed = true
 
                         /*  wait for more data  */
                         if (!self.closing)
@@ -300,10 +320,6 @@ export default class SpeechFlowNodeT2TSentence extends SpeechFlowNode {
             clearTimeout(this.workingOffTimer)
             this.workingOffTimer = null
         }
-        if (this.previewTimer !== null) {
-            clearTimeout(this.previewTimer)
-            this.previewTimer = null
-        }
 
         /*  remove any pending event listeners  */
         this.queue.removeAllListeners("write")

package/speechflow-cli/src/speechflow-util-queue.ts

@@ -97,6 +97,7 @@ export type QueueElement = { type: string }
 export class QueuePointer<T extends QueueElement> extends EventEmitter {
     /*  internal state  */
     private index = 0
+    private silence = false
 
     /*  construction  */
     constructor (
@@ -107,6 +108,17 @@ export class QueuePointer<T extends QueueElement> extends EventEmitter {
         this.setMaxListeners(100)
     }
 
+    /*  control silence operation  */
+    silent (silence: boolean) {
+        this.silence = silence
+    }
+
+    /*  notify about operation  */
+    notify (event: string, info: any) {
+        if (!this.silence)
+            this.emit(event, info)
+    }
+
     /*  positioning operations  */
     maxPosition () {
         return this.queue.elements.length
@@ -114,7 +126,7 @@ export class QueuePointer<T extends QueueElement> extends EventEmitter {
     position (index?: number): number {
         if (index !== undefined) {
             this.index = Math.max(0, Math.min(index, this.queue.elements.length))
-            this.emit("position", this.index)
+            this.notify("position", this.index)
         }
         return this.index
     }
@@ -125,19 +137,19 @@ export class QueuePointer<T extends QueueElement> extends EventEmitter {
         else if (num < 0)
             this.index = Math.max(this.index + num, 0)
         if (this.index !== indexOld)
-            this.emit("position", { start: this.index })
+            this.notify("position", { start: this.index })
     }
     walkForwardUntil (type: T["type"]) {
         while (this.index < this.queue.elements.length
             && this.queue.elements[this.index].type !== type)
             this.index++
-        this.emit("position", { start: this.index })
+        this.notify("position", { start: this.index })
     }
     walkBackwardUntil (type: T["type"]) {
         while (this.index > 0
             && this.queue.elements[this.index].type !== type)
             this.index--
-        this.emit("position", { start: this.index })
+        this.notify("position", { start: this.index })
     }
 
     /*  search operations  */
@@ -146,7 +158,7 @@ export class QueuePointer<T extends QueueElement> extends EventEmitter {
         while (position < this.queue.elements.length
             && this.queue.elements[position].type !== type)
             position++
-        this.emit("search", { start: this.index, end: position })
+        this.notify("search", { start: this.index, end: position })
         return position
     }
     searchBackward (type: T["type"]) {
@@ -154,24 +166,24 @@ export class QueuePointer<T extends QueueElement> extends EventEmitter {
         while (position > 0
             && this.queue.elements[position].type !== type)
             position--
-        this.emit("search", { start: position, end: this.index })
+        this.notify("search", { start: position, end: this.index })
         return position
     }
 
     /*  reading operations  */
-    peek (position?: number) {
+    peek (position?: number): T | undefined {
         if (position === undefined)
             position = this.index
         position = Math.max(0, Math.min(position, this.queue.elements.length))
         const element = this.queue.elements[position]
-        this.queue.emit("read", { start: position, end: position })
+        this.queue.notify("read", { start: position, end: position })
         return element
     }
-    read () {
+    read (): T | undefined {
         const element = this.queue.elements[this.index]
         if (this.index < this.queue.elements.length)
             this.index++
-        this.queue.emit("read", { start: this.index - 1, end: this.index - 1 })
+        this.queue.notify("read", { start: this.index - 1, end: this.index - 1 })
         return element
     }
     slice (size?: number) {
@@ -186,30 +198,32 @@ export class QueuePointer<T extends QueueElement> extends EventEmitter {
             slice = this.queue.elements.slice(this.index)
             this.index = this.queue.elements.length
         }
-        this.queue.emit("read", { start, end: this.index })
+        this.queue.notify("read", { start, end: this.index })
         return slice
     }
 
     /*  writing operations  */
-    touch () {
-        if (this.index >= this.queue.elements.length)
-            throw new Error("cannot touch after last element")
-        this.queue.emit("write", { start: this.index, end: this.index + 1 })
+    touch (position?: number) {
+        if (position === undefined)
+            position = this.index
+        if (position >= this.queue.elements.length)
+            throw new Error(`cannot touch after last element ${position} ${this.queue.elements.length}`)
+        this.queue.notify("write", { start: position, end: position, op: "touch" })
     }
     append (element: T) {
         this.queue.elements.push(element)
         this.index = this.queue.elements.length
-        this.queue.emit("write", { start: this.index - 1, end: this.index - 1 })
+        this.queue.notify("write", { start: this.index - 1, end: this.index - 1, op: "append" })
     }
     insert (element: T) {
         this.queue.elements.splice(this.index, 0, element)
-        this.queue.emit("write", { start: this.index - 1, end: this.index })
+        this.queue.notify("write", { start: this.index, end: this.index, op: "insert" })
     }
     delete () {
         if (this.index >= this.queue.elements.length)
             throw new Error("cannot delete after last element")
         this.queue.elements.splice(this.index, 1)
-        this.queue.emit("write", { start: this.index, end: this.index })
+        this.queue.notify("write", { start: this.index, end: this.index, op: "delete" })
     }
 }
 
@@ -217,10 +231,18 @@ export class QueuePointer<T extends QueueElement> extends EventEmitter {
 export class Queue<T extends QueueElement> extends EventEmitter {
     public elements: T[] = []
     private pointers = new Map<string, QueuePointer<T>>()
+    private silence = false
     constructor () {
         super()
         this.setMaxListeners(100)
     }
+    silent (silence: boolean) {
+        this.silence = silence
+    }
+    notify (event: string, info: any) {
+        if (!this.silence)
+            this.emit(event, info)
+    }
     pointerUse (name: string): QueuePointer<T> {
         if (!this.pointers.has(name))
             this.pointers.set(name, new QueuePointer<T>(name, this))
@@ -234,9 +256,10 @@ export class Queue<T extends QueueElement> extends EventEmitter {
     trim (): void {
         /*  determine minimum pointer position  */
         let min = this.elements.length
-        for (const pointer of this.pointers.values())
+        for (const pointer of this.pointers.values()) {
             if (min > pointer.position())
                 min = pointer.position()
+        }
 
         /*  trim the maximum amount of first elements  */
         if (min > 0) {
@@ -245,6 +268,8 @@ export class Queue<T extends QueueElement> extends EventEmitter {
             /*  shift all pointers  */
             for (const pointer of this.pointers.values())
                 pointer.position(pointer.position() - min)
+
+            this.notify("write", { start: 0, end: min, op: "trim" })
         }
     }
 }