react-native-executorch 0.4.7 → 0.5.0

package/lib/module/utils/SpeechToTextModule/ASR.js

@@ -0,0 +1,191 @@
+"use strict";
+
+// NOTE: This will be implemented in C++
+
+import { TokenizerModule } from '../../modules/natural_language_processing/TokenizerModule';
+import { ResourceFetcher } from '../ResourceFetcher';
+export class ASR {
+  tokenizerModule = new TokenizerModule();
+  timePrecision = 0.02; // Whisper timestamp precision
+  maxDecodeLength = 128;
+  chunkSize = 30; // 30 seconds
+  minChunkSamples = 1 * 16000; // 1 second
+  samplingRate = 16000;
+  async load(model, onDownloadProgressCallback) {
+    const tokenizerLoadPromise = this.tokenizerModule.load(model);
+    const encoderDecoderPromise = ResourceFetcher.fetch(onDownloadProgressCallback, model.encoderSource, model.decoderSource);
+    const [_, encoderDecoderResults] = await Promise.all([tokenizerLoadPromise, encoderDecoderPromise]);
+    const encoderSource = encoderDecoderResults?.[0];
+    const decoderSource = encoderDecoderResults?.[1];
+    if (!encoderSource || !decoderSource) {
+      throw new Error('Download interrupted.');
+    }
+    this.nativeModule = await global.loadSpeechToText(encoderSource, decoderSource, 'whisper');
+    this.startOfTranscriptToken = await this.tokenizerModule.tokenToId('<|startoftranscript|>');
+    this.endOfTextToken = await this.tokenizerModule.tokenToId('<|endoftext|>');
+    this.timestampBeginToken = await this.tokenizerModule.tokenToId('<|0.00|>');
+  }
+  async getInitialSequence(options) {
+    const initialSequence = [this.startOfTranscriptToken];
+    if (options.language) {
+      const languageToken = await this.tokenizerModule.tokenToId(`<|${options.language}|>`);
+      const taskToken = await this.tokenizerModule.tokenToId('<|transcribe|>');
+      initialSequence.push(languageToken);
+      initialSequence.push(taskToken);
+    }
+    initialSequence.push(this.timestampBeginToken);
+    return initialSequence;
+  }
+  async generate(audio, temperature, options) {
+    await this.encode(new Float32Array(audio));
+    const initialSequence = await this.getInitialSequence(options);
+    const sequencesIds = [...initialSequence];
+    const scores = [];
+    while (sequencesIds.length <= this.maxDecodeLength) {
+      const logits = this.softmaxWithTemperature(Array.from(await this.decode(sequencesIds)), temperature === 0 ? 1 : temperature);
+      const nextTokenId = temperature === 0 ? logits.indexOf(Math.max(...logits)) : this.sampleFromDistribution(logits);
+      const nextTokenProb = logits[nextTokenId];
+      sequencesIds.push(nextTokenId);
+      scores.push(nextTokenProb);
+      if (nextTokenId === this.endOfTextToken) {
+        break;
+      }
+    }
+    return {
+      sequencesIds: sequencesIds.slice(initialSequence.length),
+      scores: scores.slice(initialSequence.length)
+    };
+  }
+  softmaxWithTemperature(logits, temperature = 1.0) {
+    const max = Math.max(...logits);
+    const exps = logits.map(logit => Math.exp((logit - max) / temperature));
+    const sum = exps.reduce((a, b) => a + b, 0);
+    return exps.map(exp => exp / sum);
+  }
+  sampleFromDistribution(probs) {
+    const r = Math.random();
+    let cumulative = 0;
+    for (let i = 0; i < probs.length; i++) {
+      cumulative += probs[i];
+      if (r < cumulative) {
+        return i;
+      }
+    }
+    return probs.length - 1;
+  }
+  async generateWithFallback(audio, options) {
+    const temperatures = [0.0, 0.2, 0.4, 0.6, 0.8, 1.0];
+    let generatedTokens = [];
+    for (const temperature of temperatures) {
+      const result = await this.generate(audio, temperature, options);
+      const tokens = result.sequencesIds;
+      const scores = result.scores;
+      const seqLen = tokens.length;
+      const cumLogProb = scores.reduce((acc, score) => acc + Math.log(score), 0);
+      const avgLogProb = cumLogProb / seqLen;
+      if (avgLogProb >= -1.0) {
+        generatedTokens = tokens;
+        break;
+      }
+    }
+    return this.calculateWordLevelTimestamps(generatedTokens, audio);
+  }
+  async calculateWordLevelTimestamps(generatedTokens, audio) {
+    const segments = [];
+    let tokens = [];
+    let prevTimestamp = this.timestampBeginToken;
+    for (let i = 0; i < generatedTokens.length; i++) {
+      if (generatedTokens[i] < this.timestampBeginToken) {
+        tokens.push(generatedTokens[i]);
+      }
+      if (i > 0 && generatedTokens[i - 1] >= this.timestampBeginToken && generatedTokens[i] >= this.timestampBeginToken) {
+        const start = prevTimestamp;
+        const end = generatedTokens[i - 1];
+        const wordObjects = await this.estimateWordTimestampsLinear(tokens, start, end);
+        segments.push({
+          words: wordObjects
+        });
+        tokens = [];
+        prevTimestamp = generatedTokens[i];
+      }
+    }
+    const start = prevTimestamp;
+    const end = generatedTokens.at(-2);
+    const wordObjects = await this.estimateWordTimestampsLinear(tokens, start, end);
+    segments.push({
+      words: wordObjects
+    });
+    const scalingFactor = audio.length / this.samplingRate / ((end - this.timestampBeginToken) * this.timePrecision);
+    if (scalingFactor < 1) {
+      for (const segment of segments) {
+        for (const word of segment.words) {
+          word.start *= scalingFactor;
+          word.end *= scalingFactor;
+        }
+      }
+    }
+    return segments;
+  }
+  async estimateWordTimestampsLinear(tokens, start, end) {
+    const duration = (end - start) * this.timePrecision;
+    const segmentText = (await this.tokenizerModule.decode(tokens)).trim();
+    const words = segmentText.split(' ').map(w => ` ${w}`);
+    const numOfCharacters = words.reduce((acc, word) => acc + word.length, 0);
+    const timePerCharacter = duration / numOfCharacters;
+    const wordObjects = [];
+    const startTimeOffset = (start - this.timestampBeginToken) * this.timePrecision;
+    let prevCharNum = 0;
+    for (let j = 0; j < words.length; j++) {
+      const word = words[j];
+      const start = startTimeOffset + prevCharNum * timePerCharacter;
+      const end = start + timePerCharacter * word.length;
+      wordObjects.push({
+        word,
+        start,
+        end
+      });
+      prevCharNum += word.length;
+    }
+    return wordObjects;
+  }
+  async transcribe(audio, options) {
+    let seek = 0;
+    const allSegments = [];
+    while (seek * this.samplingRate < audio.length) {
+      const chunk = audio.slice(seek * this.samplingRate, (seek + this.chunkSize) * this.samplingRate);
+      if (chunk.length < this.minChunkSamples) {
+        return allSegments;
+      }
+      const segments = await this.generateWithFallback(chunk, options);
+      for (const segment of segments) {
+        for (const word of segment.words) {
+          word.start += seek;
+          word.end += seek;
+        }
+      }
+      allSegments.push(...segments);
+      const lastTimeStamp = segments.at(-1).words.at(-1).end;
+      seek = lastTimeStamp;
+    }
+    return allSegments;
+  }
+  tsWords(segments) {
+    const o = [];
+    for (const segment of segments) {
+      for (const word of segment.words) {
+        o.push([word.start, word.end, word.word]);
+      }
+    }
+    return o;
+  }
+  segmentsEndTs(res) {
+    return res.map(segment => segment.words.at(-1).end);
+  }
+  async encode(waveform) {
+    await this.nativeModule.encode(waveform);
+  }
+  async decode(tokens) {
+    return new Float32Array(await this.nativeModule.decode(tokens));
+  }
+}
+//# sourceMappingURL=ASR.js.map

package/lib/module/utils/SpeechToTextModule/ASR.js.map

@@ -0,0 +1 @@
+{"version":3,"names":["TokenizerModule","ResourceFetcher","ASR","tokenizerModule","timePrecision","maxDecodeLength","chunkSize","minChunkSamples","samplingRate","load","model","onDownloadProgressCallback","tokenizerLoadPromise","encoderDecoderPromise","fetch","encoderSource","decoderSource","_","encoderDecoderResults","Promise","all","Error","nativeModule","global","loadSpeechToText","startOfTranscriptToken","tokenToId","endOfTextToken","timestampBeginToken","getInitialSequence","options","initialSequence","language","languageToken","taskToken","push","generate","audio","temperature","encode","Float32Array","sequencesIds","scores","length","logits","softmaxWithTemperature","Array","from","decode","nextTokenId","indexOf","Math","max","sampleFromDistribution","nextTokenProb","slice","exps","map","logit","exp","sum","reduce","a","b","probs","r","random","cumulative","i","generateWithFallback","temperatures","generatedTokens","result","tokens","seqLen","cumLogProb","acc","score","log","avgLogProb","calculateWordLevelTimestamps","segments","prevTimestamp","start","end","wordObjects","estimateWordTimestampsLinear","words","at","scalingFactor","segment","word","duration","segmentText","trim","split","w","numOfCharacters","timePerCharacter","startTimeOffset","prevCharNum","j","transcribe","seek","allSegments","chunk","lastTimeStamp","tsWords","o","segmentsEndTs","res","waveform"],"sourceRoot":"../../../../src","sources":["utils/SpeechToTextModule/ASR.ts"],"mappings":";;AAAA;;AAEA,SAASA,eAAe,QAAQ,2DAA2D;AAQ3F,SAASC,eAAe,QAAQ,oBAAoB;AAEpD,OAAO,MAAMC,GAAG,CAAC;EAEPC,eAAe,GAAoB,IAAIH,eAAe,CAAC,CAAC;EAExDI,aAAa,GAAW,IAAI,CAAC,CAAC;EAC9BC,eAAe,GAAW,GAAG;EAC7BC,SAAS,GAAW,EAAE,CAAC,CAAC;EACxBC,eAAe,GAAW,CAAC,GAAG,KAAK,CAAC,CAAC;EACrCC,YAAY,GAAW,KAAK;EAMpC,MAAaC,IAAIA,CACfC,KAA8B,EAC9BC,0BAAsD,EACtD;IACA,MAAMC,oBAAoB,GAAG,IAAI,CAACT,eAAe,CAACM,IAAI,CAACC,KAAK,CAAC;IAC7D,MAAMG,qBAAqB,GAAGZ,eAAe,CAACa,KAAK,CACjDH,0BAA0B,EAC1BD,KAAK,CAACK,aAAa,EACnBL,KAAK,CAACM,aACR,CAAC;IACD,MAAM,CAACC,CAAC,EAAEC,qBAAqB,CAAC,GAAG,MAAMC,OAAO,CAACC,GAAG,CAAC,CACnDR,oBAAoB,EACpBC,qBAAqB,CACtB,CAAC;IACF,MAAME,aAAa,GAAGG,qBAAqB,GAAG,CAAC,CAAC;IAChD,MAAMF,aAAa,GAAGE,qBAAqB,GAAG,CAAC,CAAC;IAChD,IAAI,CAACH,aAAa,IAAI,CAACC,aAAa,EAAE;MACpC,MAAM,IAAIK,KAAK,CAAC,uBAAuB,CAAC;IAC1C;IACA,IAAI,CAACC,YAAY,GAAG,MAAMC,MAAM,CAACC,gBAAgB,CAC/CT,aAAa,EACbC,aAAa,EACb,SACF,CAAC;IAED,IAAI,CAACS,sBAAsB,GAAG,MAAM,IAAI,CAACtB,eAAe,CAACuB,SAAS,CAChE,uBACF,CAAC;IACD,IAAI,CAACC,cAAc,GAAG,MAAM,IAAI,CAACxB,eAAe,CAACuB,SAAS,CAAC,eAAe,CAAC;IAC3E,IAAI,CAACE,mBAAmB,GAAG,MAAM,IAAI,CAACzB,eAAe,CAACuB,SAAS,CAAC,UAAU,CAAC;EAC7E;EAEA,MAAcG,kBAAkBA,CAC9BC,OAAwB,EACL;IACnB,MAAMC,eAAyB,GAAG,CAAC,IAAI,CAACN,sBAAsB,CAAC;IAC/D,IAAIK,OAAO,CAACE,QAAQ,EAAE;MACpB,MAAMC,aAAa,GAAG,MAAM,IAAI,CAAC9B,eAAe,CAACuB,SAAS,CACxD,KAAKI,OAAO,CAACE,QAAQ,IACvB,CAAC;MACD,MAAME,SAAS,GAAG,MAAM,IAAI,CAAC/B,eAAe,CAACuB,SAAS,CAAC,gBAAgB,CAAC;MACxEK,eAAe,CAACI,IAAI,CAACF,aAAa,CAAC;MACnCF,eAAe,CAACI,IAAI,CAACD,SAAS,CAAC;IACjC;IACAH,eAAe,CAACI,IAAI,CAAC,IAAI,CAACP,mBAAmB,CAAC;IAC9C,OAAOG,eAAe;EACxB;EAEA,MAAcK,QAAQA,CACpBC,KAAe,EACfC,WAAmB,EACnBR,OAAwB,EAIvB;IACD,MAAM,IAAI,CAACS,MAAM,CAAC,IAAIC,YAAY,CAACH,KAAK,CAAC,CAAC;IAC1C,MAAMN,eAAe,GAAG,MAAM,IAAI,CAACF,kBAAkB,CAACC,OAAO,CAAC;IAC9D,MAAMW,YAAY,GAAG,CAAC,GAAGV,eAAe,CAAC;IACzC,MAAMW,MAAgB,GAAG,EAAE;IAE3B,OAAOD,YAAY,CAACE,MAAM,IAAI,IAAI,CAACtC,eAAe,EAAE;MAClD,MAAMuC,MAAM,GAAG,IAAI,CAACC,sBAAsB,CACxCC,KAAK,CAACC,IAAI,CAAC,MAAM,IAAI,CAACC,MAAM,CAACP,YAAY,CAAC,CAAC,EAC3CH,WAAW,KAAK,CAAC,GAAG,CAAC,GAAGA,WAC1B,CAAC;MACD,MAAMW,WAAW,GACfX,WAAW,KAAK,CAAC,GACbM,MAAM,CAACM,OAAO,CAACC,IAAI,CAACC,GAAG,CAAC,GAAGR,MAAM,CAAC,CAAC,GACnC,IAAI,CAACS,sBAAsB,CAACT,MAAM,CAAC;MACzC,MAAMU,aAAa,GAAGV,MAAM,CAACK,WAAW,CAAE;MAC1CR,YAAY,CAACN,IAAI,CAACc,WAAW,CAAC;MAC9BP,MAAM,CAACP,IAAI,CAACmB,aAAa,CAAC;MAC1B,IAAIL,WAAW,KAAK,IAAI,CAACtB,cAAc,EAAE;QACvC;MACF;IACF;IAEA,OAAO;MACLc,YAAY,EAAEA,YAAY,CAACc,KAAK,CAACxB,eAAe,CAACY,MAAM,CAAC;MACxDD,MAAM,EAAEA,MAAM,CAACa,KAAK,CAACxB,eAAe,CAACY,MAAM;IAC7C,CAAC;EACH;EAEQE,sBAAsBA,CAACD,MAAgB,EAAEN,WAAW,GAAG,GAAG,EAAE;IAClE,MAAMc,GAAG,GAAGD,IAAI,CAACC,GAAG,CAAC,GAAGR,MAAM,CAAC;IAC/B,MAAMY,IAAI,GAAGZ,MAAM,CAACa,GAAG,CAAEC,KAAK,IAAKP,IAAI,CAACQ,GAAG,CAAC,CAACD,KAAK,GAAGN,GAAG,IAAId,WAAW,CAAC,CAAC;IACzE,MAAMsB,GAAG,GAAGJ,IAAI,CAACK,MAAM,CAAC,CAACC,CAAC,EAAEC,CAAC,KAAKD,CAAC,GAAGC,CAAC,EAAE,CAAC,CAAC;IAC3C,OAAOP,IAAI,CAACC,GAAG,CAAEE,GAAG,IAAKA,GAAG,GAAGC,GAAG,CAAC;EACrC;EAEQP,sBAAsBA,CAACW,KAAe,EAAU;IACtD,MAAMC,CAAC,GAAGd,IAAI,CAACe,MAAM,CAAC,CAAC;IACvB,IAAIC,UAAU,GAAG,CAAC;IAClB,KAAK,IAAIC,CAAC,GAAG,CAAC,EAAEA,CAAC,GAAGJ,KAAK,CAACrB,MAAM,EAAEyB,CAAC,EAAE,EAAE;MACrCD,UAAU,IAAIH,KAAK,CAACI,CAAC,CAAE;MACvB,IAAIH,CAAC,GAAGE,UAAU,EAAE;QAClB,OAAOC,CAAC;MACV;IACF;IACA,OAAOJ,KAAK,CAACrB,MAAM,GAAG,CAAC;EACzB;EAEA,MAAc0B,oBAAoBA,CAChChC,KAAe,EACfP,OAAwB,EACxB;IACA,MAAMwC,YAAY,GAAG,CAAC,GAAG,EAAE,GAAG,EAAE,GAAG,EAAE,GAAG,EAAE,GAAG,EAAE,GAAG,CAAC;IACnD,IAAIC,eAAyB,GAAG,EAAE;IAElC,KAAK,MAAMjC,WAAW,IAAIgC,YAAY,EAAE;MACtC,MAAME,MAAM,GAAG,MAAM,IAAI,CAACpC,QAAQ,CAACC,KAAK,EAAEC,WAAW,EAAER,OAAO,CAAC;MAC/D,MAAM2C,MAAM,GAAGD,MAAM,CAAC/B,YAAY;MAClC,MAAMC,MAAM,GAAG8B,MAAM,CAAC9B,MAAM;MAE5B,MAAMgC,MAAM,GAAGD,MAAM,CAAC9B,MAAM;MAC5B,MAAMgC,UAAU,GAAGjC,MAAM,CAACmB,MAAM,CAC9B,CAACe,GAAG,EAAEC,KAAK,KAAKD,GAAG,GAAGzB,IAAI,CAAC2B,GAAG,CAACD,KAAK,CAAC,EACrC,CACF,CAAC;MACD,MAAME,UAAU,GAAGJ,UAAU,GAAGD,MAAM;MAEtC,IAAIK,UAAU,IAAI,CAAC,GAAG,EAAE;QACtBR,eAAe,GAAGE,MAAM;QACxB;MACF;IACF;IAEA,OAAO,IAAI,CAACO,4BAA4B,CAACT,eAAe,EAAElC,KAAK,CAAC;EAClE;EAEA,MAAc2C,4BAA4BA,CACxCT,eAAyB,EACzBlC,KAAe,EACK;IACpB,MAAM4C,QAAmB,GAAG,EAAE;IAE9B,IAAIR,MAAgB,GAAG,EAAE;IACzB,IAAIS,aAAa,GAAG,IAAI,CAACtD,mBAAmB;IAC5C,KAAK,IAAIwC,CAAC,GAAG,CAAC,EAAEA,CAAC,GAAGG,eAAe,CAAC5B,MAAM,EAAEyB,CAAC,EAAE,EAAE;MAC/C,IAAIG,eAAe,CAACH,CAAC,CAAC,GAAI,IAAI,CAACxC,mBAAmB,EAAE;QAClD6C,MAAM,CAACtC,IAAI,CAACoC,eAAe,CAACH,CAAC,CAAE,CAAC;MAClC;MAEA,IACEA,CAAC,GAAG,CAAC,IACLG,eAAe,CAACH,CAAC,GAAG,CAAC,CAAC,IAAK,IAAI,CAACxC,mBAAmB,IACnD2C,eAAe,CAACH,CAAC,CAAC,IAAK,IAAI,CAACxC,mBAAmB,EAC/C;QACA,MAAMuD,KAAK,GAAGD,aAAa;QAC3B,MAAME,GAAG,GAAGb,eAAe,CAACH,CAAC,GAAG,CAAC,CAAE;QACnC,MAAMiB,WAAW,GAAG,MAAM,IAAI,CAACC,4BAA4B,CACzDb,MAAM,EACNU,KAAK,EACLC,GACF,CAAC;QACDH,QAAQ,CAAC9C,IAAI,CAAC;UACZoD,KAAK,EAAEF;QACT,CAAC,CAAC;QACFZ,MAAM,GAAG,EAAE;QACXS,aAAa,GAAGX,eAAe,CAACH,CAAC,CAAE;MACrC;IACF;IAEA,MAAMe,KAAK,GAAGD,aAAa;IAC3B,MAAME,GAAG,GAAGb,eAAe,CAACiB,EAAE,CAAC,CAAC,CAAC,CAAE;IACnC,MAAMH,WAAW,GAAG,MAAM,IAAI,CAACC,4BAA4B,CACzDb,MAAM,EACNU,KAAK,EACLC,GACF,CAAC;IACDH,QAAQ,CAAC9C,IAAI,CAAC;MACZoD,KAAK,EAAEF;IACT,CAAC,CAAC;IAEF,MAAMI,aAAa,GACjBpD,KAAK,CAACM,MAAM,GACZ,IAAI,CAACnC,YAAY,IAChB,CAAC4E,GAAG,GAAG,IAAI,CAACxD,mBAAmB,IAAI,IAAI,CAACxB,aAAa,CAAC;IACzD,IAAIqF,aAAa,GAAG,CAAC,EAAE;MACrB,KAAK,MAAMC,OAAO,IAAIT,QAAQ,EAAE;QAC9B,KAAK,MAAMU,IAAI,IAAID,OAAO,CAACH,KAAK,EAAE;UAChCI,IAAI,CAACR,KAAK,IAAIM,aAAa;UAC3BE,IAAI,CAACP,GAAG,IAAIK,aAAa;QAC3B;MACF;IACF;IAEA,OAAOR,QAAQ;EACjB;EAEA,MAAcK,4BAA4BA,CACxCb,MAAgB,EAChBU,KAAa,EACbC,GAAW,EACY;IACvB,MAAMQ,QAAQ,GAAG,CAACR,GAAG,GAAGD,KAAK,IAAI,IAAI,CAAC/E,aAAa;IACnD,MAAMyF,WAAW,GAAG,CACjB,MAAM,IAAI,CAAC1F,eAAe,CAAC6C,MAAM,CAACyB,MAAM,CAAC,EAC1CqB,IAAI,CAAC,CAAC;IAER,MAAMP,KAAK,GAAGM,WAAW,CAACE,KAAK,CAAC,GAAG,CAAC,CAACtC,GAAG,CAAEuC,CAAC,IAAK,IAAIA,CAAC,EAAE,CAAC;IACxD,MAAMC,eAAe,GAAGV,KAAK,CAAC1B,MAAM,CAClC,CAACe,GAAW,EAAEe,IAAY,KAAKf,GAAG,GAAGe,IAAI,CAAChD,MAAM,EAChD,CACF,CAAC;IAED,MAAMuD,gBAAgB,GAAGN,QAAQ,GAAGK,eAAe;IAEnD,MAAMZ,WAAyB,GAAG,EAAE;IACpC,MAAMc,eAAe,GACnB,CAAChB,KAAK,GAAG,IAAI,CAACvD,mBAAmB,IAAI,IAAI,CAACxB,aAAa;IAEzD,IAAIgG,WAAW,GAAG,CAAC;IACnB,KAAK,IAAIC,CAAC,GAAG,CAAC,EAAEA,CAAC,GAAGd,KAAK,CAAC5C,MAAM,EAAE0D,CAAC,EAAE,EAAE;MACrC,MAAMV,IAAI,GAAGJ,KAAK,CAACc,CAAC,CAAE;MACtB,MAAMlB,KAAK,GAAGgB,eAAe,GAAGC,WAAW,GAAGF,gBAAgB;MAC9D,MAAMd,GAAG,GAAGD,KAAK,GAAGe,gBAAgB,GAAGP,IAAI,CAAChD,MAAM;MAClD0C,WAAW,CAAClD,IAAI,CAAC;QAAEwD,IAAI;QAAER,KAAK;QAAEC;MAAI,CAAC,CAAC;MACtCgB,WAAW,IAAIT,IAAI,CAAChD,MAAM;IAC5B;IAEA,OAAO0C,WAAW;EACpB;EAEA,MAAaiB,UAAUA,CACrBjE,KAAe,EACfP,OAAwB,EACJ;IACpB,IAAIyE,IAAI,GAAG,CAAC;IACZ,MAAMC,WAAsB,GAAG,EAAE;IAEjC,OAAOD,IAAI,GAAG,IAAI,CAAC/F,YAAY,GAAG6B,KAAK,CAACM,MAAM,EAAE;MAC9C,MAAM8D,KAAK,GAAGpE,KAAK,CAACkB,KAAK,CACvBgD,IAAI,GAAG,IAAI,CAAC/F,YAAY,EACxB,CAAC+F,IAAI,GAAG,IAAI,CAACjG,SAAS,IAAI,IAAI,CAACE,YACjC,CAAC;MACD,IAAIiG,KAAK,CAAC9D,MAAM,GAAG,IAAI,CAACpC,eAAe,EAAE;QACvC,OAAOiG,WAAW;MACpB;MACA,MAAMvB,QAAQ,GAAG,MAAM,IAAI,CAACZ,oBAAoB,CAACoC,KAAK,EAAE3E,OAAO,CAAC;MAChE,KAAK,MAAM4D,OAAO,IAAIT,QAAQ,EAAE;QAC9B,KAAK,MAAMU,IAAI,IAAID,OAAO,CAACH,KAAK,EAAE;UAChCI,IAAI,CAACR,KAAK,IAAIoB,IAAI;UAClBZ,IAAI,CAACP,GAAG,IAAImB,IAAI;QAClB;MACF;MACAC,WAAW,CAACrE,IAAI,CAAC,GAAG8C,QAAQ,CAAC;MAC7B,MAAMyB,aAAa,GAAGzB,QAAQ,CAACO,EAAE,CAAC,CAAC,CAAC,CAAC,CAAED,KAAK,CAACC,EAAE,CAAC,CAAC,CAAC,CAAC,CAAEJ,GAAG;MACxDmB,IAAI,GAAGG,aAAa;IACtB;IAEA,OAAOF,WAAW;EACpB;EAEOG,OAAOA,CAAC1B,QAAmB,EAAe;IAC/C,MAAM2B,CAAc,GAAG,EAAE;IACzB,KAAK,MAAMlB,OAAO,IAAIT,QAAQ,EAAE;MAC9B,KAAK,MAAMU,IAAI,IAAID,OAAO,CAACH,KAAK,EAAE;QAChCqB,CAAC,CAACzE,IAAI,CAAC,CAACwD,IAAI,CAACR,KAAK,EAAEQ,IAAI,CAACP,GAAG,EAAEO,IAAI,CAACA,IAAI,CAAC,CAAC;MAC3C;IACF;IACA,OAAOiB,CAAC;EACV;EAEOC,aAAaA,CAACC,GAAc,EAAE;IACnC,OAAOA,GAAG,CAACrD,GAAG,CAAEiC,OAAO,IAAKA,OAAO,CAACH,KAAK,CAACC,EAAE,CAAC,CAAC,CAAC,CAAC,CAAEJ,GAAG,CAAC;EACxD;EAEA,MAAa7C,MAAMA,CAACwE,QAAsB,EAAiB;IACzD,MAAM,IAAI,CAACzF,YAAY,CAACiB,MAAM,CAACwE,QAAQ,CAAC;EAC1C;EAEA,MAAa/D,MAAMA,CAACyB,MAAgB,EAAyB;IAC3D,OAAO,IAAIjC,YAAY,CAAC,MAAM,IAAI,CAAClB,YAAY,CAAC0B,MAAM,CAACyB,MAAM,CAAC,CAAC;EACjE;AACF","ignoreList":[]}

package/lib/module/utils/SpeechToTextModule/OnlineProcessor.js

@@ -0,0 +1,73 @@
+"use strict";
+
+// NOTE: This will be implemented in C++
+
+import { HypothesisBuffer } from './hypothesisBuffer';
+export class OnlineASRProcessor {
+  samplingRate = 16000;
+  audioBuffer = [];
+  transcriptBuffer = new HypothesisBuffer();
+  bufferTimeOffset = 0;
+  committed = [];
+  constructor(asr) {
+    this.asr = asr;
+  }
+  insertAudioChunk(audio) {
+    this.audioBuffer.push(...audio);
+  }
+  async processIter(options) {
+    const res = await this.asr.transcribe(this.audioBuffer, options);
+    const tsw = this.asr.tsWords(res);
+    this.transcriptBuffer.insert(tsw, this.bufferTimeOffset);
+    const o = this.transcriptBuffer.flush();
+    this.committed.push(...o);
+    const s = 15;
+    if (this.audioBuffer.length / this.samplingRate > s) {
+      this.chunkCompletedSegment(res);
+    }
+    const committed = this.toFlush(o)[2];
+    const nonCommitted = this.transcriptBuffer.complete().map(x => x[2]).join('');
+    return {
+      committed,
+      nonCommitted
+    };
+  }
+  chunkCompletedSegment(res) {
+    if (this.committed.length === 0) {
+      return;
+    }
+    const ends = this.asr.segmentsEndTs(res);
+    const t = this.committed.at(-1)[1];
+    if (ends.length > 1) {
+      let e = ends.at(-2) + this.bufferTimeOffset;
+      while (ends.length > 2 && e > t) {
+        ends.pop();
+        e = ends.at(-2) + this.bufferTimeOffset;
+      }
+      if (e <= t) {
+        this.chunkAt(e);
+      }
+    }
+  }
+  chunkAt(time) {
+    this.transcriptBuffer.popCommitted(time);
+    const cutSeconds = time - this.bufferTimeOffset;
+    this.audioBuffer = this.audioBuffer.slice(Math.floor(cutSeconds * this.samplingRate));
+    this.bufferTimeOffset = time;
+  }
+  async finish() {
+    const o = this.transcriptBuffer.complete();
+    const f = this.toFlush(o);
+    this.bufferTimeOffset += this.audioBuffer.length / this.samplingRate;
+    return {
+      committed: f[2]
+    };
+  }
+  toFlush(words) {
+    const t = words.map(s => s[2]).join(' ');
+    const b = words.length === 0 ? null : words[0][0];
+    const e = words.length === 0 ? null : words.at(-1)[1];
+    return [b, e, t];
+  }
+}
+//# sourceMappingURL=OnlineProcessor.js.map

package/lib/module/utils/SpeechToTextModule/OnlineProcessor.js.map

@@ -0,0 +1 @@
+{"version":3,"names":["HypothesisBuffer","OnlineASRProcessor","samplingRate","audioBuffer","transcriptBuffer","bufferTimeOffset","committed","constructor","asr","insertAudioChunk","audio","push","processIter","options","res","transcribe","tsw","tsWords","insert","o","flush","s","length","chunkCompletedSegment","toFlush","nonCommitted","complete","map","x","join","ends","segmentsEndTs","t","at","e","pop","chunkAt","time","popCommitted","cutSeconds","slice","Math","floor","finish","f","words","b"],"sourceRoot":"../../../../src","sources":["utils/SpeechToTextModule/OnlineProcessor.ts"],"mappings":";;AAAA;;AAIA,SAASA,gBAAgB,QAAQ,oBAAoB;AAErD,OAAO,MAAMC,kBAAkB,CAAC;EAGtBC,YAAY,GAAW,KAAK;EAC7BC,WAAW,GAAa,EAAE;EACzBC,gBAAgB,GAAqB,IAAIJ,gBAAgB,CAAC,CAAC;EAC3DK,gBAAgB,GAAW,CAAC;EAC5BC,SAAS,GAAgB,EAAE;EAEnCC,WAAWA,CAACC,GAAQ,EAAE;IACpB,IAAI,CAACA,GAAG,GAAGA,GAAG;EAChB;EAEOC,gBAAgBA,CAACC,KAAe,EAAE;IACvC,IAAI,CAACP,WAAW,CAACQ,IAAI,CAAC,GAAGD,KAAK,CAAC;EACjC;EAEA,MAAaE,WAAWA,CAACC,OAAwB,EAAE;IACjD,MAAMC,GAAG,GAAG,MAAM,IAAI,CAACN,GAAG,CAACO,UAAU,CAAC,IAAI,CAACZ,WAAW,EAAEU,OAAO,CAAC;IAChE,MAAMG,GAAG,GAAG,IAAI,CAACR,GAAG,CAACS,OAAO,CAACH,GAAG,CAAC;IACjC,IAAI,CAACV,gBAAgB,CAACc,MAAM,CAACF,GAAG,EAAE,IAAI,CAACX,gBAAgB,CAAC;IACxD,MAAMc,CAAC,GAAG,IAAI,CAACf,gBAAgB,CAACgB,KAAK,CAAC,CAAC;IACvC,IAAI,CAACd,SAAS,CAACK,IAAI,CAAC,GAAGQ,CAAC,CAAC;IAEzB,MAAME,CAAC,GAAG,EAAE;IACZ,IAAI,IAAI,CAAClB,WAAW,CAACmB,MAAM,GAAG,IAAI,CAACpB,YAAY,GAAGmB,CAAC,EAAE;MACnD,IAAI,CAACE,qBAAqB,CAACT,GAAG,CAAC;IACjC;IAEA,MAAMR,SAAS,GAAG,IAAI,CAACkB,OAAO,CAACL,CAAC,CAAC,CAAC,CAAC,CAAC;IACpC,MAAMM,YAAY,GAAG,IAAI,CAACrB,gBAAgB,CACvCsB,QAAQ,CAAC,CAAC,CACVC,GAAG,CAAEC,CAAC,IAAKA,CAAC,CAAC,CAAC,CAAC,CAAC,CAChBC,IAAI,CAAC,EAAE,CAAC;IACX,OAAO;MAAEvB,SAAS;MAAEmB;IAAa,CAAC;EACpC;EAEQF,qBAAqBA,CAACT,GAAc,EAAE;IAC5C,IAAI,IAAI,CAACR,SAAS,CAACgB,MAAM,KAAK,CAAC,EAAE;MAC/B;IACF;IAEA,MAAMQ,IAAI,GAAG,IAAI,CAACtB,GAAG,CAACuB,aAAa,CAACjB,GAAG,CAAC;IACxC,MAAMkB,CAAC,GAAG,IAAI,CAAC1B,SAAS,CAAC2B,EAAE,CAAC,CAAC,CAAC,CAAC,CAAE,CAAC,CAAC;IAEnC,IAAIH,IAAI,CAACR,MAAM,GAAG,CAAC,EAAE;MACnB,IAAIY,CAAC,GAAGJ,IAAI,CAACG,EAAE,CAAC,CAAC,CAAC,CAAC,GAAI,IAAI,CAAC5B,gBAAgB;MAC5C,OAAOyB,IAAI,CAACR,MAAM,GAAG,CAAC,IAAIY,CAAC,GAAGF,CAAC,EAAE;QAC/BF,IAAI,CAACK,GAAG,CAAC,CAAC;QACVD,CAAC,GAAGJ,IAAI,CAACG,EAAE,CAAC,CAAC,CAAC,CAAC,GAAI,IAAI,CAAC5B,gBAAgB;MAC1C;MAEA,IAAI6B,CAAC,IAAIF,CAAC,EAAE;QACV,IAAI,CAACI,OAAO,CAACF,CAAC,CAAC;MACjB;IACF;EACF;EAEQE,OAAOA,CAACC,IAAY,EAAE;IAC5B,IAAI,CAACjC,gBAAgB,CAACkC,YAAY,CAACD,IAAI,CAAC;IACxC,MAAME,UAAU,GAAGF,IAAI,GAAG,IAAI,CAAChC,gBAAgB;IAC/C,IAAI,CAACF,WAAW,GAAG,IAAI,CAACA,WAAW,CAACqC,KAAK,CACvCC,IAAI,CAACC,KAAK,CAACH,UAAU,GAAG,IAAI,CAACrC,YAAY,CAC3C,CAAC;IACD,IAAI,CAACG,gBAAgB,GAAGgC,IAAI;EAC9B;EAEA,MAAaM,MAAMA,CAAA,EAAG;IACpB,MAAMxB,CAAC,GAAG,IAAI,CAACf,gBAAgB,CAACsB,QAAQ,CAAC,CAAC;IAC1C,MAAMkB,CAAC,GAAG,IAAI,CAACpB,OAAO,CAACL,CAAC,CAAC;IACzB,IAAI,CAACd,gBAAgB,IAAI,IAAI,CAACF,WAAW,CAACmB,MAAM,GAAG,IAAI,CAACpB,YAAY;IACpE,OAAO;MAAEI,SAAS,EAAEsC,CAAC,CAAC,CAAC;IAAE,CAAC;EAC5B;EAEQpB,OAAOA,CAACqB,KAAkB,EAA0C;IAC1E,MAAMb,CAAC,GAAGa,KAAK,CAAClB,GAAG,CAAEN,CAAC,IAAKA,CAAC,CAAC,CAAC,CAAC,CAAC,CAACQ,IAAI,CAAC,GAAG,CAAC;IAC1C,MAAMiB,CAAC,GAAGD,KAAK,CAACvB,MAAM,KAAK,CAAC,GAAG,IAAI,GAAGuB,KAAK,CAAC,CAAC,CAAC,CAAE,CAAC,CAAC;IAClD,MAAMX,CAAC,GAAGW,KAAK,CAACvB,MAAM,KAAK,CAAC,GAAG,IAAI,GAAGuB,KAAK,CAACZ,EAAE,CAAC,CAAC,CAAC,CAAC,CAAE,CAAC,CAAC;IACtD,OAAO,CAACa,CAAC,EAAEZ,CAAC,EAAEF,CAAC,CAAC;EAClB;AACF","ignoreList":[]}

package/lib/module/utils/SpeechToTextModule/hypothesisBuffer.js

@@ -0,0 +1,56 @@
+"use strict";
+
+// NOTE: This will be implemented in C++
+
+export class HypothesisBuffer {
+  committedInBuffer = [];
+  buffer = [];
+  new = [];
+  lastCommittedTime = 0;
+  lastCommittedWord = null;
+  insert(newWords, offset) {
+    const newWordsOffset = newWords.map(([a, b, t]) => [a + offset, b + offset, t]);
+    this.new = newWordsOffset.filter(([a, _b, _t]) => a > this.lastCommittedTime - 0.5);
+    if (this.new.length > 0) {
+      const [a, _b, _t] = this.new[0];
+      if (Math.abs(a - this.lastCommittedTime) < 1 && this.committedInBuffer.length > 0) {
+        const cn = this.committedInBuffer.length;
+        const nn = this.new.length;
+        for (let i = 1; i <= Math.min(cn, nn, 5); i++) {
+          const c = this.committedInBuffer.slice(-i).map(w => w[2]).join(' ');
+          const tail = this.new.slice(0, i).map(w => w[2]).join(' ');
+          if (c === tail) {
+            for (let j = 0; j < i; j++) {
+              this.new.shift();
+            }
+            break;
+          }
+        }
+      }
+    }
+  }
+  flush() {
+    const commit = [];
+    while (this.new.length > 0 && this.buffer.length > 0) {
+      if (this.new[0][2] !== this.buffer[0][2]) {
+        break;
+      }
+      commit.push(this.new[0]);
+      this.lastCommittedWord = this.new[0][2];
+      this.lastCommittedTime = this.new[0][1];
+      this.buffer.shift();
+      this.new.shift();
+    }
+    this.buffer = this.new;
+    this.new = [];
+    this.committedInBuffer.push(...commit);
+    return commit;
+  }
+  popCommitted(time) {
+    this.committedInBuffer = this.committedInBuffer.filter(([_a, b, _t]) => b > time);
+  }
+  complete() {
+    return this.buffer;
+  }
+}
+//# sourceMappingURL=hypothesisBuffer.js.map

package/lib/module/utils/SpeechToTextModule/hypothesisBuffer.js.map

@@ -0,0 +1 @@
+{"version":3,"names":["HypothesisBuffer","committedInBuffer","buffer","new","lastCommittedTime","lastCommittedWord","insert","newWords","offset","newWordsOffset","map","a","b","t","filter","_b","_t","length","Math","abs","cn","nn","i","min","c","slice","w","join","tail","j","shift","flush","commit","push","popCommitted","time","_a","complete"],"sourceRoot":"../../../../src","sources":["utils/SpeechToTextModule/hypothesisBuffer.ts"],"mappings":";;AAAA;;AAIA,OAAO,MAAMA,gBAAgB,CAAC;EACpBC,iBAAiB,GAAgB,EAAE;EACnCC,MAAM,GAAgB,EAAE;EACxBC,GAAG,GAAgB,EAAE;EAErBC,iBAAiB,GAAW,CAAC;EAC9BC,iBAAiB,GAAkB,IAAI;EAEvCC,MAAMA,CAACC,QAAqB,EAAEC,MAAc,EAAE;IACnD,MAAMC,cAA2B,GAAGF,QAAQ,CAACG,GAAG,CAAC,CAAC,CAACC,CAAC,EAAEC,CAAC,EAAEC,CAAC,CAAC,KAAK,CAC9DF,CAAC,GAAGH,MAAM,EACVI,CAAC,GAAGJ,MAAM,EACVK,CAAC,CACF,CAAC;IACF,IAAI,CAACV,GAAG,GAAGM,cAAc,CAACK,MAAM,CAC9B,CAAC,CAACH,CAAC,EAAEI,EAAE,EAAEC,EAAE,CAAC,KAAKL,CAAC,GAAG,IAAI,CAACP,iBAAiB,GAAG,GAChD,CAAC;IAED,IAAI,IAAI,CAACD,GAAG,CAACc,MAAM,GAAG,CAAC,EAAE;MACvB,MAAM,CAACN,CAAC,EAAEI,EAAE,EAAEC,EAAE,CAAC,GAAG,IAAI,CAACb,GAAG,CAAC,CAAC,CAAE;MAChC,IACEe,IAAI,CAACC,GAAG,CAACR,CAAC,GAAG,IAAI,CAACP,iBAAiB,CAAC,GAAG,CAAC,IACxC,IAAI,CAACH,iBAAiB,CAACgB,MAAM,GAAG,CAAC,EACjC;QACA,MAAMG,EAAE,GAAG,IAAI,CAACnB,iBAAiB,CAACgB,MAAM;QACxC,MAAMI,EAAE,GAAG,IAAI,CAAClB,GAAG,CAACc,MAAM;QAE1B,KAAK,IAAIK,CAAC,GAAG,CAAC,EAAEA,CAAC,IAAIJ,IAAI,CAACK,GAAG,CAACH,EAAE,EAAEC,EAAE,EAAE,CAAC,CAAC,EAAEC,CAAC,EAAE,EAAE;UAC7C,MAAME,CAAC,GAAG,IAAI,CAACvB,iBAAiB,CAC7BwB,KAAK,CAAC,CAACH,CAAC,CAAC,CACTZ,GAAG,CAAEgB,CAAC,IAAKA,CAAC,CAAC,CAAC,CAAC,CAAC,CAChBC,IAAI,CAAC,GAAG,CAAC;UACZ,MAAMC,IAAI,GAAG,IAAI,CAACzB,GAAG,CAClBsB,KAAK,CAAC,CAAC,EAAEH,CAAC,CAAC,CACXZ,GAAG,CAAEgB,CAAC,IAAKA,CAAC,CAAC,CAAC,CAAC,CAAC,CAChBC,IAAI,CAAC,GAAG,CAAC;UACZ,IAAIH,CAAC,KAAKI,IAAI,EAAE;YACd,KAAK,IAAIC,CAAC,GAAG,CAAC,EAAEA,CAAC,GAAGP,CAAC,EAAEO,CAAC,EAAE,EAAE;cAC1B,IAAI,CAAC1B,GAAG,CAAC2B,KAAK,CAAC,CAAC;YAClB;YACA;UACF;QACF;MACF;IACF;EACF;EAEOC,KAAKA,CAAA,EAAgB;IAC1B,MAAMC,MAAmB,GAAG,EAAE;IAC9B,OAAO,IAAI,CAAC7B,GAAG,CAACc,MAAM,GAAG,CAAC,IAAI,IAAI,CAACf,MAAM,CAACe,MAAM,GAAG,CAAC,EAAE;MACpD,IAAI,IAAI,CAACd,GAAG,CAAC,CAAC,CAAC,CAAE,CAAC,CAAC,KAAK,IAAI,CAACD,MAAM,CAAC,CAAC,CAAC,CAAE,CAAC,CAAC,EAAE;QAC1C;MACF;MACA8B,MAAM,CAACC,IAAI,CAAC,IAAI,CAAC9B,GAAG,CAAC,CAAC,CAAE,CAAC;MACzB,IAAI,CAACE,iBAAiB,GAAG,IAAI,CAACF,GAAG,CAAC,CAAC,CAAC,CAAE,CAAC,CAAC;MACxC,IAAI,CAACC,iBAAiB,GAAG,IAAI,CAACD,GAAG,CAAC,CAAC,CAAC,CAAE,CAAC,CAAC;MACxC,IAAI,CAACD,MAAM,CAAC4B,KAAK,CAAC,CAAC;MACnB,IAAI,CAAC3B,GAAG,CAAC2B,KAAK,CAAC,CAAC;IAClB;IACA,IAAI,CAAC5B,MAAM,GAAG,IAAI,CAACC,GAAG;IACtB,IAAI,CAACA,GAAG,GAAG,EAAE;IACb,IAAI,CAACF,iBAAiB,CAACgC,IAAI,CAAC,GAAGD,MAAM,CAAC;IACtC,OAAOA,MAAM;EACf;EAEOE,YAAYA,CAACC,IAAY,EAAE;IAChC,IAAI,CAAClC,iBAAiB,GAAG,IAAI,CAACA,iBAAiB,CAACa,MAAM,CACpD,CAAC,CAACsB,EAAE,EAAExB,CAAC,EAAEI,EAAE,CAAC,KAAKJ,CAAC,GAAGuB,IACvB,CAAC;EACH;EAEOE,QAAQA,CAAA,EAAgB;IAC7B,OAAO,IAAI,CAACnC,MAAM;EACpB;AACF","ignoreList":[]}

package/lib/module/utils/llm.js

@@ -1,5 +1,11 @@
 "use strict";
 
+import * as z from 'zod/v4';
+import { Validator } from 'jsonschema';
+import { jsonrepair } from 'jsonrepair';
+import { DEFAULT_STRUCTURED_OUTPUT_PROMPT } from '../constants/llmDefaults';
+import * as zCore from 'zod/v4/core';
+import { Logger } from '../common/Logger';
 export const parseToolCall = message => {
   try {
     const unparsedToolCalls = message.match('\\[(.|\\s)*\\]');
@@ -18,8 +24,42 @@ export const parseToolCall = message => {
     }
     return results;
   } catch (e) {
-    console.error(e);
+    Logger.error(e);
     return [];
   }
 };
+const filterObjectKeys = (obj, keysToRemove) => {
+  const entries = Object.entries(obj);
+  const filteredEntries = entries.filter(([key, _]) => !keysToRemove.includes(key));
+  return Object.fromEntries(filteredEntries);
+};
+export const getStructuredOutputPrompt = responseSchema => {
+  const schemaObject = responseSchema instanceof zCore.$ZodType ? filterObjectKeys(z.toJSONSchema(responseSchema), ['$schema', 'additionalProperties']) : responseSchema;
+  const schemaString = JSON.stringify(schemaObject);
+  return DEFAULT_STRUCTURED_OUTPUT_PROMPT(schemaString);
+};
+const extractBetweenBrackets = text => {
+  const startIndex = text.search(/[\\{\\[]/); // First occurrence of either { or [
+
+  const openingBracket = text[startIndex];
+  const closingBracket = openingBracket === '{' ? '}' : ']';
+  if (!openingBracket) throw Error("Couldn't find JSON in text");
+  return text.slice(text.indexOf(openingBracket), text.lastIndexOf(closingBracket) + 1);
+};
+
+// this is a bit hacky typing
+export const fixAndValidateStructuredOutput = (output, responseSchema) => {
+  const extractedOutput = extractBetweenBrackets(output);
+  const repairedOutput = jsonrepair(extractedOutput);
+  const outputJSON = JSON.parse(repairedOutput);
+  if (responseSchema instanceof zCore.$ZodType) {
+    return z.parse(responseSchema, outputJSON);
+  } else {
+    const validator = new Validator();
+    validator.validate(outputJSON, responseSchema, {
+      throwAll: true
+    });
+    return outputJSON;
+  }
+};
 //# sourceMappingURL=llm.js.map

package/lib/module/utils/llm.js.map

@@ -1 +1 @@
-{"version":3,"names":["parseToolCall","message","unparsedToolCalls","match","Error","parsedMessage","JSON","parse","results","tool","name","arguments","push","toolName","e","console","error"],"sourceRoot":"../../../src","sources":["utils/llm.ts"],"mappings":";;AAEA,OAAO,MAAMA,aAA8C,GACzDC,OAAe,IACZ;EACH,IAAI;IACF,MAAMC,iBAAiB,GAAGD,OAAO,CAACE,KAAK,CAAC,gBAAgB,CAAC;IACzD,IAAI,CAACD,iBAAiB,EAAE;MACtB,MAAME,KAAK,CAAC,4BAA4B,CAAC;IAC3C;IACA,MAAMC,aAAwB,GAAGC,IAAI,CAACC,KAAK,CAACL,iBAAiB,CAAC,CAAC,CAAC,CAAC;IACjE,MAAMM,OAAO,GAAG,EAAE;IAElB,KAAK,MAAMC,IAAI,IAAIJ,aAAa,EAAE;MAChC,IACE,MAAM,IAAII,IAAI,IACd,OAAOA,IAAI,CAACC,IAAI,KAAK,QAAQ,IAC7B,WAAW,IAAID,IAAI,IACnBA,IAAI,CAACE,SAAS,KAAK,IAAI,IACvB,OAAOF,IAAI,CAACE,SAAS,KAAK,QAAQ,EAClC;QACAH,OAAO,CAACI,IAAI,CAAC;UACXC,QAAQ,EAAEJ,IAAI,CAACC,IAAI;UACnBC,SAAS,EAAEF,IAAI,CAACE;QAClB,CAAC,CAAC;MACJ;IACF;IAEA,OAAOH,OAAO;EAChB,CAAC,CAAC,OAAOM,CAAC,EAAE;IACVC,OAAO,CAACC,KAAK,CAACF,CAAC,CAAC;IAChB,OAAO,EAAE;EACX;AACF,CAAC","ignoreList":[]}
+{"version":3,"names":["z","Validator","jsonrepair","DEFAULT_STRUCTURED_OUTPUT_PROMPT","zCore","Logger","parseToolCall","message","unparsedToolCalls","match","Error","parsedMessage","JSON","parse","results","tool","name","arguments","push","toolName","e","error","filterObjectKeys","obj","keysToRemove","entries","Object","filteredEntries","filter","key","_","includes","fromEntries","getStructuredOutputPrompt","responseSchema","schemaObject","$ZodType","toJSONSchema","schemaString","stringify","extractBetweenBrackets","text","startIndex","search","openingBracket","closingBracket","slice","indexOf","lastIndexOf","fixAndValidateStructuredOutput","output","extractedOutput","repairedOutput","outputJSON","validator","validate","throwAll"],"sourceRoot":"../../../src","sources":["utils/llm.ts"],"mappings":";;AACA,OAAO,KAAKA,CAAC,MAAM,QAAQ;AAC3B,SAAiBC,SAAS,QAAQ,YAAY;AAC9C,SAASC,UAAU,QAAQ,YAAY;AACvC,SAASC,gCAAgC,QAAQ,0BAA0B;AAC3E,OAAO,KAAKC,KAAK,MAAM,aAAa;AACpC,SAASC,MAAM,QAAQ,kBAAkB;AAEzC,OAAO,MAAMC,aAA8C,GACzDC,OAAe,IACZ;EACH,IAAI;IACF,MAAMC,iBAAiB,GAAGD,OAAO,CAACE,KAAK,CAAC,gBAAgB,CAAC;IACzD,IAAI,CAACD,iBAAiB,EAAE;MACtB,MAAME,KAAK,CAAC,4BAA4B,CAAC;IAC3C;IACA,MAAMC,aAAwB,GAAGC,IAAI,CAACC,KAAK,CAACL,iBAAiB,CAAC,CAAC,CAAC,CAAC;IACjE,MAAMM,OAAO,GAAG,EAAE;IAElB,KAAK,MAAMC,IAAI,IAAIJ,aAAa,EAAE;MAChC,IACE,MAAM,IAAII,IAAI,IACd,OAAOA,IAAI,CAACC,IAAI,KAAK,QAAQ,IAC7B,WAAW,IAAID,IAAI,IACnBA,IAAI,CAACE,SAAS,KAAK,IAAI,IACvB,OAAOF,IAAI,CAACE,SAAS,KAAK,QAAQ,EAClC;QACAH,OAAO,CAACI,IAAI,CAAC;UACXC,QAAQ,EAAEJ,IAAI,CAACC,IAAI;UACnBC,SAAS,EAAEF,IAAI,CAACE;QAClB,CAAC,CAAC;MACJ;IACF;IAEA,OAAOH,OAAO;EAChB,CAAC,CAAC,OAAOM,CAAC,EAAE;IACVf,MAAM,CAACgB,KAAK,CAACD,CAAC,CAAC;IACf,OAAO,EAAE;EACX;AACF,CAAC;AAED,MAAME,gBAAgB,GAAGA,CAACC,GAAW,EAAEC,YAAsB,KAAK;EAChE,MAAMC,OAAO,GAAGC,MAAM,CAACD,OAAO,CAACF,GAAG,CAAC;EACnC,MAAMI,eAAe,GAAGF,OAAO,CAACG,MAAM,CACpC,CAAC,CAACC,GAAG,EAAEC,CAAC,CAAC,KAAK,CAACN,YAAY,CAACO,QAAQ,CAACF,GAAG,CAC1C,CAAC;EACD,OAAOH,MAAM,CAACM,WAAW,CAACL,eAAe,CAAC;AAC5C,CAAC;AAED,OAAO,MAAMM,yBAAyB,GACpCC,cAA0B,IACvB;EACH,MAAMC,YAAkD,GACtDD,cAAc,YAAY9B,KAAK,CAACgC,QAAQ,GACpCd,gBAAgB,CAACtB,CAAC,CAACqC,YAAY,CAACH,cAAc,CAAC,EAAE,CAC/C,SAAS,EACT,sBAAsB,CACvB,CAAC,GACFA,cAAc;EAEpB,MAAMI,YAAY,GAAG1B,IAAI,CAAC2B,SAAS,CAACJ,YAAY,CAAC;EAEjD,OAAOhC,gCAAgC,CAACmC,YAAY,CAAC;AACvD,CAAC;AAED,MAAME,sBAAsB,GAAIC,IAAY,IAAa;EACvD,MAAMC,UAAU,GAAGD,IAAI,CAACE,MAAM,CAAC,UAAU,CAAC,CAAC,CAAC;;EAE5C,MAAMC,cAAc,GAAGH,IAAI,CAACC,UAAU,CAAC;EACvC,MAAMG,cAAc,GAAGD,cAAc,KAAK,GAAG,GAAG,GAAG,GAAG,GAAG;EAEzD,IAAI,CAACA,cAAc,EAAE,MAAMlC,KAAK,CAAC,4BAA4B,CAAC;EAE9D,OAAO+B,IAAI,CAACK,KAAK,CACfL,IAAI,CAACM,OAAO,CAACH,cAAc,CAAC,EAC5BH,IAAI,CAACO,WAAW,CAACH,cAAc,CAAC,GAAG,CACrC,CAAC;AACH,CAAC;;AAED;AACA,OAAO,MAAMI,8BAA8B,GAAGA,CAC5CC,MAAc,EACdhB,cAA0B,KACN;EACpB,MAAMiB,eAAe,GAAGX,sBAAsB,CAACU,MAAM,CAAC;EACtD,MAAME,cAAc,GAAGlD,UAAU,CAACiD,eAAe,CAAC;EAClD,MAAME,UAAU,GAAGzC,IAAI,CAACC,KAAK,CAACuC,cAAc,CAAC;EAE7C,IAAIlB,cAAc,YAAY9B,KAAK,CAACgC,QAAQ,EAAE;IAC5C,OAAOpC,CAAC,CAACa,KAAK,CAACqB,cAAc,EAAEmB,UAAU,CAAC;EAC5C,CAAC,MAAM;IACL,MAAMC,SAAS,GAAG,IAAIrD,SAAS,CAAC,CAAC;IACjCqD,SAAS,CAACC,QAAQ,CAACF,UAAU,EAAEnB,cAAc,EAAE;MAC7CsB,QAAQ,EAAE;IACZ,CAAC,CAAC;IACF,OAAOH,UAAU;EACnB;AACF,CAAC","ignoreList":[]}

package/lib/typescript/Error.d.ts

@@ -21,6 +21,8 @@ export declare enum ETError {
     MemoryAllocationFailed = 33,
     AccessFailed = 34,
     InvalidProgram = 35,
+    InvalidExternalData = 36,
+    OutOfResources = 37,
     DelegateInvalidCompatibility = 48,
     DelegateMemoryAllocationFailed = 49,
     DelegateInvalidHandle = 50

package/lib/typescript/Error.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"Error.d.ts","sourceRoot":"","sources":["../../src/Error.ts"],"names":[],"mappings":"AAAA,oBAAY,OAAO;IAEjB,cAAc,MAAO;IACrB,eAAe,MAAO;IACtB,eAAe,MAAO;IACtB,eAAe,MAAO;IACtB,oBAAoB,MAAO;IAC3B,kBAAkB,MAAO;IAGzB,yBAAyB,MAAO;IAChC,gBAAgB,MAAO;IACvB,mBAAmB,MAAO;IAK1B,EAAE,IAAO;IACT,QAAQ,IAAO;IACf,YAAY,IAAO;IACnB,WAAW,IAAO;IAGlB,YAAY,KAAO;IACnB,cAAc,KAAO;IACrB,eAAe,KAAO;IACtB,WAAW,KAAO;IAClB,eAAe,KAAO;IAGtB,QAAQ,KAAO;IACf,sBAAsB,KAAO;IAC7B,YAAY,KAAO;IACnB,cAAc,KAAO;IAGrB,4BAA4B,KAAO;IACnC,8BAA8B,KAAO;IACrC,qBAAqB,KAAO;CAC7B;AAED,eAAO,MAAM,QAAQ,GAAI,GAAG,OAAO,GAAG,OAAO,GAAG,KAAK,KAAG,MAkBvD,CAAC"}
+{"version":3,"file":"Error.d.ts","sourceRoot":"","sources":["../../src/Error.ts"],"names":[],"mappings":"AAAA,oBAAY,OAAO;IAEjB,cAAc,MAAO;IACrB,eAAe,MAAO;IACtB,eAAe,MAAO;IACtB,eAAe,MAAO;IACtB,oBAAoB,MAAO;IAC3B,kBAAkB,MAAO;IAGzB,yBAAyB,MAAO;IAChC,gBAAgB,MAAO;IACvB,mBAAmB,MAAO;IAK1B,EAAE,IAAO;IACT,QAAQ,IAAO;IACf,YAAY,IAAO;IACnB,WAAW,IAAO;IAGlB,YAAY,KAAO;IACnB,cAAc,KAAO;IACrB,eAAe,KAAO;IACtB,WAAW,KAAO;IAClB,eAAe,KAAO;IAGtB,QAAQ,KAAO;IACf,sBAAsB,KAAO;IAC7B,YAAY,KAAO;IACnB,cAAc,KAAO;IACrB,mBAAmB,KAAO;IAC1B,cAAc,KAAO;IAGrB,4BAA4B,KAAO;IACnC,8BAA8B,KAAO;IACrC,qBAAqB,KAAO;CAC7B;AAED,eAAO,MAAM,QAAQ,GAAI,GAAG,OAAO,GAAG,OAAO,GAAG,KAAK,KAAG,MAcvD,CAAC"}

package/lib/typescript/common/Logger.d.ts

@@ -0,0 +1,9 @@
+export declare class Logger {
+    private static readonly PREFIX;
+    static log(...data: any[]): void;
+    static debug(...data: any[]): void;
+    static info(...data: any[]): void;
+    static warn(...data: any[]): void;
+    static error(...data: any[]): void;
+}
+//# sourceMappingURL=Logger.d.ts.map

package/lib/typescript/common/Logger.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"Logger.d.ts","sourceRoot":"","sources":["../../../src/common/Logger.ts"],"names":[],"mappings":"AAEA,qBAAa,MAAM;IACjB,OAAO,CAAC,MAAM,CAAC,QAAQ,CAAC,MAAM,CAA+B;IAE7D,MAAM,CAAC,GAAG,CAAC,GAAG,IAAI,EAAE,GAAG,EAAE;IAIzB,MAAM,CAAC,KAAK,CAAC,GAAG,IAAI,EAAE,GAAG,EAAE;IAI3B,MAAM,CAAC,IAAI,CAAC,GAAG,IAAI,EAAE,GAAG,EAAE;IAI1B,MAAM,CAAC,IAAI,CAAC,GAAG,IAAI,EAAE,GAAG,EAAE;IAI1B,MAAM,CAAC,KAAK,CAAC,GAAG,IAAI,EAAE,GAAG,EAAE;CAG5B"}

package/lib/typescript/constants/llmDefaults.d.ts

@@ -1,5 +1,6 @@
 import { ChatConfig, Message } from '../types/llm';
 export declare const DEFAULT_SYSTEM_PROMPT = "You are a knowledgeable, efficient, and direct AI assistant. Provide concise answers, focusing on the key information needed. Offer suggestions tactfully when appropriate to improve outcomes. Engage in productive collaboration with the user. Don't return too much text.";
+export declare const DEFAULT_STRUCTURED_OUTPUT_PROMPT: (structuredOutputSchema: string) => string;
 export declare const DEFAULT_MESSAGE_HISTORY: Message[];
 export declare const DEFAULT_CONTEXT_WINDOW_LENGTH = 5;
 export declare const DEFAULT_CHAT_CONFIG: ChatConfig;

package/lib/typescript/constants/llmDefaults.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"llmDefaults.d.ts","sourceRoot":"","sources":["../../../src/constants/llmDefaults.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,UAAU,EAAE,OAAO,EAAE,MAAM,cAAc,CAAC;AAEnD,eAAO,MAAM,qBAAqB,kRAC+O,CAAC;AAElR,eAAO,MAAM,uBAAuB,EAAE,OAAO,EAAO,CAAC;AAErD,eAAO,MAAM,6BAA6B,IAAI,CAAC;AAE/C,eAAO,MAAM,mBAAmB,EAAE,UAIjC,CAAC"}
+{"version":3,"file":"llmDefaults.d.ts","sourceRoot":"","sources":["../../../src/constants/llmDefaults.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,UAAU,EAAE,OAAO,EAAE,MAAM,cAAc,CAAC;AAEnD,eAAO,MAAM,qBAAqB,kRAC+O,CAAC;AAElR,eAAO,MAAM,gCAAgC,GAC3C,wBAAwB,MAAM,WAQ/B,CAAC;AAEF,eAAO,MAAM,uBAAuB,EAAE,OAAO,EAAO,CAAC;AAErD,eAAO,MAAM,6BAA6B,IAAI,CAAC;AAE/C,eAAO,MAAM,mBAAmB,EAAE,UAIjC,CAAC"}