react-native-executorch 0.5.3 → 0.5.4

package/src/utils/SpeechToTextModule/ASR.ts

@@ -1,303 +0,0 @@
-// NOTE: This will be implemented in C++
-
-import { TokenizerModule } from '../../modules/natural_language_processing/TokenizerModule';
-import {
-  DecodingOptions,
-  Segment,
-  SpeechToTextModelConfig,
-  WordObject,
-  WordTuple,
-} from '../../types/stt';
-import { ResourceFetcher } from '../ResourceFetcher';
-
-export class ASR {
-  private nativeModule: any;
-  private tokenizerModule: TokenizerModule = new TokenizerModule();
-
-  private timePrecision: number = 0.02; // Whisper timestamp precision
-  private maxDecodeLength: number = 128;
-  private chunkSize: number = 30; // 30 seconds
-  private minChunkSamples: number = 1 * 16000; // 1 second
-  private samplingRate: number = 16000;
-
-  private startOfTranscriptToken!: number;
-  private endOfTextToken!: number;
-  private timestampBeginToken!: number;
-
-  public async load(
-    model: SpeechToTextModelConfig,
-    onDownloadProgressCallback: (progress: number) => void
-  ) {
-    const tokenizerLoadPromise = this.tokenizerModule.load(model);
-    const encoderDecoderPromise = ResourceFetcher.fetch(
-      onDownloadProgressCallback,
-      model.encoderSource,
-      model.decoderSource
-    );
-    const [_, encoderDecoderResults] = await Promise.all([
-      tokenizerLoadPromise,
-      encoderDecoderPromise,
-    ]);
-    const encoderSource = encoderDecoderResults?.[0];
-    const decoderSource = encoderDecoderResults?.[1];
-    if (!encoderSource || !decoderSource) {
-      throw new Error('Download interrupted.');
-    }
-    this.nativeModule = await global.loadSpeechToText(
-      encoderSource,
-      decoderSource,
-      'whisper'
-    );
-
-    this.startOfTranscriptToken = await this.tokenizerModule.tokenToId(
-      '<|startoftranscript|>'
-    );
-    this.endOfTextToken = await this.tokenizerModule.tokenToId('<|endoftext|>');
-    this.timestampBeginToken = await this.tokenizerModule.tokenToId('<|0.00|>');
-  }
-
-  private async getInitialSequence(
-    options: DecodingOptions
-  ): Promise<number[]> {
-    const initialSequence: number[] = [this.startOfTranscriptToken];
-    if (options.language) {
-      const languageToken = await this.tokenizerModule.tokenToId(
-        `<|${options.language}|>`
-      );
-      const taskToken = await this.tokenizerModule.tokenToId('<|transcribe|>');
-      initialSequence.push(languageToken);
-      initialSequence.push(taskToken);
-    }
-    initialSequence.push(this.timestampBeginToken);
-    return initialSequence;
-  }
-
-  private async generate(
-    audio: number[],
-    temperature: number,
-    options: DecodingOptions
-  ): Promise<{
-    sequencesIds: number[];
-    scores: number[];
-  }> {
-    await this.encode(new Float32Array(audio));
-    const initialSequence = await this.getInitialSequence(options);
-    const sequencesIds = [...initialSequence];
-    const scores: number[] = [];
-
-    while (sequencesIds.length <= this.maxDecodeLength) {
-      const logits = this.softmaxWithTemperature(
-        Array.from(await this.decode(sequencesIds)),
-        temperature === 0 ? 1 : temperature
-      );
-      const nextTokenId =
-        temperature === 0
-          ? logits.indexOf(Math.max(...logits))
-          : this.sampleFromDistribution(logits);
-      const nextTokenProb = logits[nextTokenId]!;
-      sequencesIds.push(nextTokenId);
-      scores.push(nextTokenProb);
-      if (nextTokenId === this.endOfTextToken) {
-        break;
-      }
-    }
-
-    return {
-      sequencesIds: sequencesIds.slice(initialSequence.length),
-      scores: scores.slice(initialSequence.length),
-    };
-  }
-
-  private softmaxWithTemperature(logits: number[], temperature = 1.0) {
-    const max = Math.max(...logits);
-    const exps = logits.map((logit) => Math.exp((logit - max) / temperature));
-    const sum = exps.reduce((a, b) => a + b, 0);
-    return exps.map((exp) => exp / sum);
-  }
-
-  private sampleFromDistribution(probs: number[]): number {
-    const r = Math.random();
-    let cumulative = 0;
-    for (let i = 0; i < probs.length; i++) {
-      cumulative += probs[i]!;
-      if (r < cumulative) {
-        return i;
-      }
-    }
-    return probs.length - 1;
-  }
-
-  private async generateWithFallback(
-    audio: number[],
-    options: DecodingOptions
-  ) {
-    const temperatures = [0.0, 0.2, 0.4, 0.6, 0.8, 1.0];
-    let generatedTokens: number[] = [];
-
-    for (const temperature of temperatures) {
-      const result = await this.generate(audio, temperature, options);
-      const tokens = result.sequencesIds;
-      const scores = result.scores;
-
-      const seqLen = tokens.length;
-      const cumLogProb = scores.reduce(
-        (acc, score) => acc + Math.log(score),
-        0
-      );
-      const avgLogProb = cumLogProb / seqLen;
-
-      if (avgLogProb >= -1.0) {
-        generatedTokens = tokens;
-        break;
-      }
-    }
-
-    return this.calculateWordLevelTimestamps(generatedTokens, audio);
-  }
-
-  private async calculateWordLevelTimestamps(
-    generatedTokens: number[],
-    audio: number[]
-  ): Promise<Segment[]> {
-    const segments: Segment[] = [];
-
-    let tokens: number[] = [];
-    let prevTimestamp = this.timestampBeginToken;
-    for (let i = 0; i < generatedTokens.length; i++) {
-      if (generatedTokens[i]! < this.timestampBeginToken) {
-        tokens.push(generatedTokens[i]!);
-      }
-
-      if (
-        i > 0 &&
-        generatedTokens[i - 1]! >= this.timestampBeginToken &&
-        generatedTokens[i]! >= this.timestampBeginToken
-      ) {
-        const start = prevTimestamp;
-        const end = generatedTokens[i - 1]!;
-        const wordObjects = await this.estimateWordTimestampsLinear(
-          tokens,
-          start,
-          end
-        );
-        segments.push({
-          words: wordObjects,
-        });
-        tokens = [];
-        prevTimestamp = generatedTokens[i]!;
-      }
-    }
-
-    const start = prevTimestamp;
-    const end = generatedTokens.at(-2)!;
-    const wordObjects = await this.estimateWordTimestampsLinear(
-      tokens,
-      start,
-      end
-    );
-    segments.push({
-      words: wordObjects,
-    });
-
-    const scalingFactor =
-      audio.length /
-      this.samplingRate /
-      ((end - this.timestampBeginToken) * this.timePrecision);
-    if (scalingFactor < 1) {
-      for (const segment of segments) {
-        for (const word of segment.words) {
-          word.start *= scalingFactor;
-          word.end *= scalingFactor;
-        }
-      }
-    }
-
-    return segments;
-  }
-
-  private async estimateWordTimestampsLinear(
-    tokens: number[],
-    timestampStart: number,
-    timestampEnd: number
-  ): Promise<WordObject[]> {
-    const duration = (timestampEnd - timestampStart) * this.timePrecision;
-    const segmentText = (
-      (await this.tokenizerModule.decode(tokens)) as string
-    ).trim();
-
-    const words = segmentText.split(' ').map((w) => ` ${w}`);
-    const numOfCharacters = words.reduce(
-      (acc: number, word: string) => acc + word.length,
-      0
-    );
-
-    const timePerCharacter = duration / numOfCharacters;
-
-    const wordObjects: WordObject[] = [];
-    const startTimeOffset =
-      (timestampStart - this.timestampBeginToken) * this.timePrecision;
-
-    let prevCharNum = 0;
-    for (let j = 0; j < words.length; j++) {
-      const word = words[j]!;
-      const start = startTimeOffset + prevCharNum * timePerCharacter;
-      const end = start + timePerCharacter * word.length;
-      wordObjects.push({ word, start, end });
-      prevCharNum += word.length;
-    }
-
-    return wordObjects;
-  }
-
-  public async transcribe(
-    audio: number[],
-    options: DecodingOptions
-  ): Promise<Segment[]> {
-    let seek = 0;
-    const allSegments: Segment[] = [];
-
-    while (seek * this.samplingRate < audio.length) {
-      const chunk = audio.slice(
-        seek * this.samplingRate,
-        (seek + this.chunkSize) * this.samplingRate
-      );
-      if (chunk.length < this.minChunkSamples) {
-        return allSegments;
-      }
-      const segments = await this.generateWithFallback(chunk, options);
-      for (const segment of segments) {
-        for (const word of segment.words) {
-          word.start += seek;
-          word.end += seek;
-        }
-      }
-      allSegments.push(...segments);
-      const lastTimeStamp = segments.at(-1)!.words.at(-1)!.end;
-      seek = lastTimeStamp;
-    }
-
-    return allSegments;
-  }
-
-  public tsWords(segments: Segment[]): WordTuple[] {
-    const o: WordTuple[] = [];
-    for (const segment of segments) {
-      for (const word of segment.words) {
-        o.push([word.start, word.end, word.word]);
-      }
-    }
-    return o;
-  }
-
-  public segmentsEndTs(res: Segment[]) {
-    return res.map((segment) => segment.words.at(-1)!.end);
-  }
-
-  public async encode(waveform: Float32Array): Promise<void> {
-    await this.nativeModule.encode(waveform);
-  }
-
-  public async decode(tokens: number[]): Promise<Float32Array> {
-    return new Float32Array(await this.nativeModule.decode(tokens));
-  }
-}

package/src/utils/SpeechToTextModule/OnlineProcessor.ts

@@ -1,87 +0,0 @@
-// NOTE: This will be implemented in C++
-
-import { WordTuple, DecodingOptions, Segment } from '../../types/stt';
-import { ASR } from './ASR';
-import { HypothesisBuffer } from './hypothesisBuffer';
-
-export class OnlineASRProcessor {
-  private asr: ASR;
-
-  private samplingRate: number = 16000;
-  public audioBuffer: number[] = [];
-  private transcriptBuffer: HypothesisBuffer = new HypothesisBuffer();
-  private bufferTimeOffset: number = 0;
-  private committed: WordTuple[] = [];
-
-  constructor(asr: ASR) {
-    this.asr = asr;
-  }
-
-  public insertAudioChunk(audio: number[]) {
-    this.audioBuffer.push(...audio);
-  }
-
-  public async processIter(options: DecodingOptions) {
-    const res = await this.asr.transcribe(this.audioBuffer, options);
-    const tsw = this.asr.tsWords(res);
-    this.transcriptBuffer.insert(tsw, this.bufferTimeOffset);
-    const o = this.transcriptBuffer.flush();
-    this.committed.push(...o);
-
-    const s = 15;
-    if (this.audioBuffer.length / this.samplingRate > s) {
-      this.chunkCompletedSegment(res);
-    }
-
-    const committed = this.toFlush(o)[2];
-    const nonCommitted = this.transcriptBuffer
-      .complete()
-      .map((x) => x[2])
-      .join('');
-    return { committed, nonCommitted };
-  }
-
-  private chunkCompletedSegment(res: Segment[]) {
-    if (this.committed.length === 0) {
-      return;
-    }
-
-    const ends = this.asr.segmentsEndTs(res);
-    const t = this.committed.at(-1)![1];
-
-    if (ends.length > 1) {
-      let e = ends.at(-2)! + this.bufferTimeOffset;
-      while (ends.length > 2 && e > t) {
-        ends.pop();
-        e = ends.at(-2)! + this.bufferTimeOffset;
-      }
-
-      if (e <= t) {
-        this.chunkAt(e);
-      }
-    }
-  }
-
-  private chunkAt(time: number) {
-    this.transcriptBuffer.popCommitted(time);
-    const cutSeconds = time - this.bufferTimeOffset;
-    this.audioBuffer = this.audioBuffer.slice(
-      Math.floor(cutSeconds * this.samplingRate)
-    );
-    this.bufferTimeOffset = time;
-  }
-
-  public async finish() {
-    const o = this.transcriptBuffer.complete();
-    const f = this.toFlush(o);
-    this.bufferTimeOffset += this.audioBuffer.length / this.samplingRate;
-    return { committed: f[2] };
-  }
-
-  private toFlush(words: WordTuple[]): [number | null, number | null, string] {
-    const t = words.map((s) => s[2]).join(' ');
-    const b = words.length === 0 ? null : words[0]![0];
-    const e = words.length === 0 ? null : words.at(-1)![1];
-    return [b, e, t];
-  }
-}

package/src/utils/SpeechToTextModule/hypothesisBuffer.ts

@@ -1,79 +0,0 @@
-// NOTE: This will be implemented in C++
-
-import { WordTuple } from '../../types/stt';
-
-export class HypothesisBuffer {
-  private committedInBuffer: WordTuple[] = [];
-  private buffer: WordTuple[] = [];
-  private new: WordTuple[] = [];
-
-  private lastCommittedTime: number = 0;
-  public lastCommittedWord: string | null = null;
-
-  public insert(newWords: WordTuple[], offset: number) {
-    const newWordsOffset: WordTuple[] = newWords.map(([a, b, t]) => [
-      a + offset,
-      b + offset,
-      t,
-    ]);
-    this.new = newWordsOffset.filter(
-      ([a, _b, _t]) => a > this.lastCommittedTime - 0.5
-    );
-
-    if (this.new.length > 0) {
-      const [a, _b, _t] = this.new[0]!;
-      if (
-        Math.abs(a - this.lastCommittedTime) < 1 &&
-        this.committedInBuffer.length > 0
-      ) {
-        const cn = this.committedInBuffer.length;
-        const nn = this.new.length;
-
-        for (let i = 1; i <= Math.min(cn, nn, 5); i++) {
-          const c = this.committedInBuffer
-            .slice(-i)
-            .map((w) => w[2])
-            .join(' ');
-          const tail = this.new
-            .slice(0, i)
-            .map((w) => w[2])
-            .join(' ');
-          if (c === tail) {
-            for (let j = 0; j < i; j++) {
-              this.new.shift();
-            }
-            break;
-          }
-        }
-      }
-    }
-  }
-
-  public flush(): WordTuple[] {
-    const commit: WordTuple[] = [];
-    while (this.new.length > 0 && this.buffer.length > 0) {
-      if (this.new[0]![2] !== this.buffer[0]![2]) {
-        break;
-      }
-      commit.push(this.new[0]!);
-      this.lastCommittedWord = this.new[0]![2];
-      this.lastCommittedTime = this.new[0]![1];
-      this.buffer.shift();
-      this.new.shift();
-    }
-    this.buffer = this.new;
-    this.new = [];
-    this.committedInBuffer.push(...commit);
-    return commit;
-  }
-
-  public popCommitted(time: number) {
-    this.committedInBuffer = this.committedInBuffer.filter(
-      ([_a, b, _t]) => b > time
-    );
-  }
-
-  public complete(): WordTuple[] {
-    return this.buffer;
-  }
-}

package/src/utils/stt.ts

@@ -1,28 +0,0 @@
-export const longCommonInfPref = (
-  seq1: number[],
-  seq2: number[],
-  hammingDistThreshold: number
-) => {
-  let maxInd = 0;
-  let maxLength = 0;
-
-  for (let i = 0; i < seq1.length; i++) {
-    let j = 0;
-    let hammingDist = 0;
-    while (
-      j < seq2.length &&
-      i + j < seq1.length &&
-      (seq1[i + j] === seq2[j] || hammingDist < hammingDistThreshold)
-    ) {
-      if (seq1[i + j] !== seq2[j]) {
-        hammingDist++;
-      }
-      j++;
-    }
-    if (j >= maxLength) {
-      maxLength = j;
-      maxInd = i;
-    }
-  }
-  return maxInd;
-};