react-native-executorch 0.4.8 → 0.5.1-rc.0

package/lib/controllers/LLMController.js

@@ -0,0 +1,210 @@
+import { ResourceFetcher } from '../utils/ResourceFetcher';
+import { ETError, getError } from '../Error';
+import { Template } from '@huggingface/jinja';
+import { DEFAULT_CHAT_CONFIG } from '../constants/llmDefaults';
+import { readAsStringAsync } from 'expo-file-system';
+import { SPECIAL_TOKENS, } from '../types/llm';
+import { parseToolCall } from '../utils/llm';
+import { Logger } from '../common/Logger';
+export class LLMController {
+    nativeModule;
+    chatConfig = DEFAULT_CHAT_CONFIG;
+    toolsConfig;
+    tokenizerConfig;
+    onToken;
+    _response = '';
+    _isReady = false;
+    _isGenerating = false;
+    _messageHistory = [];
+    // User callbacks
+    tokenCallback;
+    responseCallback;
+    messageHistoryCallback;
+    isReadyCallback;
+    isGeneratingCallback;
+    onDownloadProgressCallback;
+    constructor({ tokenCallback, responseCallback, messageHistoryCallback, isReadyCallback, isGeneratingCallback, onDownloadProgressCallback, }) {
+        if (responseCallback !== undefined) {
+            Logger.warn('Passing response callback is deprecated and will be removed in 0.6.0');
+        }
+        this.tokenCallback = (token) => {
+            tokenCallback?.(token);
+        };
+        this.responseCallback = (response) => {
+            this._response = response;
+            responseCallback?.(response);
+        };
+        this.messageHistoryCallback = (messageHistory) => {
+            this._messageHistory = messageHistory;
+            messageHistoryCallback?.(messageHistory);
+        };
+        this.isReadyCallback = (isReady) => {
+            this._isReady = isReady;
+            isReadyCallback?.(isReady);
+        };
+        this.isGeneratingCallback = (isGenerating) => {
+            this._isGenerating = isGenerating;
+            isGeneratingCallback?.(isGenerating);
+        };
+        this.onDownloadProgressCallback = onDownloadProgressCallback;
+    }
+    get response() {
+        return this._response;
+    }
+    get isReady() {
+        return this._isReady;
+    }
+    get isGenerating() {
+        return this._isGenerating;
+    }
+    get messageHistory() {
+        return this._messageHistory;
+    }
+    async load({ modelSource, tokenizerSource, tokenizerConfigSource, }) {
+        // reset inner state when loading new model
+        this.responseCallback('');
+        this.messageHistoryCallback(this.chatConfig.initialMessageHistory);
+        this.isGeneratingCallback(false);
+        this.isReadyCallback(false);
+        try {
+            const paths = await ResourceFetcher.fetch(this.onDownloadProgressCallback, tokenizerSource, tokenizerConfigSource, modelSource);
+            if (paths === null || paths?.length < 3) {
+                throw new Error('Download interrupted!');
+            }
+            const tokenizerFileUri = paths[0];
+            const tokenizerConfigFileUri = paths[1];
+            const modelFileUri = paths[2];
+            this.tokenizerConfig = JSON.parse(await readAsStringAsync('file://' + tokenizerConfigFileUri));
+            this.nativeModule = global.loadLLM(modelFileUri, tokenizerFileUri);
+            this.isReadyCallback(true);
+            this.onToken = (data) => {
+                if (!data ||
+                    (SPECIAL_TOKENS.EOS_TOKEN in this.tokenizerConfig &&
+                        data === this.tokenizerConfig.eos_token) ||
+                    (SPECIAL_TOKENS.PAD_TOKEN in this.tokenizerConfig &&
+                        data === this.tokenizerConfig.pad_token)) {
+                    return;
+                }
+                this.tokenCallback(data);
+                this.responseCallback(this._response + data);
+            };
+        }
+        catch (e) {
+            this.isReadyCallback(false);
+            throw new Error(getError(e));
+        }
+    }
+    setTokenCallback(tokenCallback) {
+        this.tokenCallback = tokenCallback;
+    }
+    configure({ chatConfig, toolsConfig, }) {
+        this.chatConfig = { ...DEFAULT_CHAT_CONFIG, ...chatConfig };
+        this.toolsConfig = toolsConfig;
+        // reset inner state when loading new configuration
+        this.responseCallback('');
+        this.messageHistoryCallback(this.chatConfig.initialMessageHistory);
+        this.isGeneratingCallback(false);
+    }
+    delete() {
+        if (this._isGenerating) {
+            throw new Error(getError(ETError.ModelGenerating) +
+                'You cannot delete the model now. You need to interrupt first.');
+        }
+        this.onToken = () => { };
+        this.nativeModule.unload();
+        this.isReadyCallback(false);
+        this.isGeneratingCallback(false);
+    }
+    async forward(input) {
+        if (!this._isReady) {
+            throw new Error(getError(ETError.ModuleNotLoaded));
+        }
+        if (this._isGenerating) {
+            throw new Error(getError(ETError.ModelGenerating));
+        }
+        try {
+            this.responseCallback('');
+            this.isGeneratingCallback(true);
+            await this.nativeModule.generate(input, this.onToken);
+        }
+        catch (e) {
+            throw new Error(getError(e));
+        }
+        finally {
+            this.isGeneratingCallback(false);
+        }
+    }
+    interrupt() {
+        this.nativeModule.interrupt();
+    }
+    async generate(messages, tools) {
+        if (!this._isReady) {
+            throw new Error(getError(ETError.ModuleNotLoaded));
+        }
+        if (messages.length === 0) {
+            throw new Error(`Empty 'messages' array!`);
+        }
+        if (messages[0] && messages[0].role !== 'system') {
+            Logger.warn(`You are not providing system prompt. You can pass it in the first message using { role: 'system', content: YOUR_PROMPT }. Otherwise prompt from your model's chat template will be used.`);
+        }
+        const renderedChat = this.applyChatTemplate(messages, this.tokenizerConfig, tools, 
+        // eslint-disable-next-line camelcase
+        { tools_in_user_message: false, add_generation_prompt: true });
+        await this.forward(renderedChat);
+    }
+    async sendMessage(message) {
+        this.messageHistoryCallback([
+            ...this._messageHistory,
+            { content: message, role: 'user' },
+        ]);
+        const messageHistoryWithPrompt = [
+            { content: this.chatConfig.systemPrompt, role: 'system' },
+            ...this._messageHistory.slice(-this.chatConfig.contextWindowLength),
+        ];
+        await this.generate(messageHistoryWithPrompt, this.toolsConfig?.tools);
+        if (!this.toolsConfig || this.toolsConfig.displayToolCalls) {
+            this.messageHistoryCallback([
+                ...this._messageHistory,
+                { content: this._response, role: 'assistant' },
+            ]);
+        }
+        if (!this.toolsConfig) {
+            return;
+        }
+        const toolCalls = parseToolCall(this._response);
+        for (const toolCall of toolCalls) {
+            this.toolsConfig
+                .executeToolCallback(toolCall)
+                .then((toolResponse) => {
+                if (toolResponse) {
+                    this.messageHistoryCallback([
+                        ...this._messageHistory,
+                        { content: toolResponse, role: 'assistant' },
+                    ]);
+                }
+            });
+        }
+    }
+    deleteMessage(index) {
+        // we delete referenced message and all messages after it
+        // so the model responses that used them are deleted as well
+        const newMessageHistory = this._messageHistory.slice(0, index);
+        this.messageHistoryCallback(newMessageHistory);
+    }
+    applyChatTemplate(messages, tokenizerConfig, tools, templateFlags) {
+        if (!tokenizerConfig.chat_template) {
+            throw Error("Tokenizer config doesn't include chat_template");
+        }
+        const template = new Template(tokenizerConfig.chat_template);
+        const specialTokens = Object.fromEntries(Object.keys(SPECIAL_TOKENS)
+            .filter((key) => key in tokenizerConfig)
+            .map((key) => [key, tokenizerConfig[key]]));
+        const result = template.render({
+            messages,
+            tools,
+            ...templateFlags,
+            ...specialTokens,
+        });
+        return result;
+    }
+}

package/lib/controllers/OCRController.js

@@ -0,0 +1,65 @@
+import { symbols } from '../constants/ocr/symbols';
+import { ETError, getError } from '../Error';
+import { ResourceFetcher } from '../utils/ResourceFetcher';
+export class OCRController {
+    nativeModule;
+    isReady = false;
+    isGenerating = false;
+    error = null;
+    modelDownloadProgressCallback;
+    isReadyCallback;
+    isGeneratingCallback;
+    errorCallback;
+    constructor({ modelDownloadProgressCallback = (_downloadProgress) => { }, isReadyCallback = (_isReady) => { }, isGeneratingCallback = (_isGenerating) => { }, errorCallback = (_error) => { }, }) {
+        this.modelDownloadProgressCallback = modelDownloadProgressCallback;
+        this.isReadyCallback = isReadyCallback;
+        this.isGeneratingCallback = isGeneratingCallback;
+        this.errorCallback = errorCallback;
+    }
+    load = async (detectorSource, recognizerSources, language) => {
+        try {
+            if (!detectorSource || Object.keys(recognizerSources).length !== 3)
+                return;
+            if (!symbols[language]) {
+                throw new Error(getError(ETError.LanguageNotSupported));
+            }
+            this.isReady = false;
+            this.isReadyCallback(false);
+            const paths = await ResourceFetcher.fetch(this.modelDownloadProgressCallback, detectorSource, recognizerSources.recognizerLarge, recognizerSources.recognizerMedium, recognizerSources.recognizerSmall);
+            if (paths === null || paths?.length < 4) {
+                throw new Error('Download interrupted!');
+            }
+            this.nativeModule = global.loadOCR(paths[0], paths[1], paths[2], paths[3], symbols[language]);
+            this.isReady = true;
+            this.isReadyCallback(this.isReady);
+        }
+        catch (e) {
+            if (this.errorCallback) {
+                this.errorCallback(getError(e));
+            }
+            else {
+                throw new Error(getError(e));
+            }
+        }
+    };
+    forward = async (input) => {
+        if (!this.isReady) {
+            throw new Error(getError(ETError.ModuleNotLoaded));
+        }
+        if (this.isGenerating) {
+            throw new Error(getError(ETError.ModelGenerating));
+        }
+        try {
+            this.isGenerating = true;
+            this.isGeneratingCallback(this.isGenerating);
+            return await this.nativeModule.generate(input);
+        }
+        catch (e) {
+            throw new Error(getError(e));
+        }
+        finally {
+            this.isGenerating = false;
+            this.isGeneratingCallback(this.isGenerating);
+        }
+    };
+}

package/lib/controllers/SpeechToTextController.d.ts

@@ -0,0 +1,52 @@
+import { MODES, STREAMING_ACTION } from '../constants/sttDefaults';
+import { AvailableModels } from '../types/stt';
+import { ResourceSource } from '../types/common';
+import { SpeechToTextLanguage } from '../types/stt';
+export declare class SpeechToTextController {
+    private speechToTextNativeModule;
+    sequence: number[];
+    isReady: boolean;
+    isGenerating: boolean;
+    private tokenizerModule;
+    private overlapSeconds;
+    private windowSize;
+    private chunks;
+    private seqs;
+    private prevSeq;
+    private waveform;
+    private numOfChunks;
+    private streaming;
+    private decodedTranscribeCallback;
+    private modelDownloadProgressCallback;
+    private isReadyCallback;
+    private isGeneratingCallback;
+    private onErrorCallback;
+    private config;
+    constructor({ transcribeCallback, modelDownloadProgressCallback, isReadyCallback, isGeneratingCallback, onErrorCallback, overlapSeconds, windowSize, streamingConfig, }: {
+        transcribeCallback: (sequence: string) => void;
+        modelDownloadProgressCallback?: (downloadProgress: number) => void;
+        isReadyCallback?: (isReady: boolean) => void;
+        isGeneratingCallback?: (isGenerating: boolean) => void;
+        onErrorCallback?: (error: Error | undefined) => void;
+        overlapSeconds?: number;
+        windowSize?: number;
+        streamingConfig?: keyof typeof MODES;
+    });
+    loadModel(modelName: AvailableModels, encoderSource?: ResourceSource, decoderSource?: ResourceSource, tokenizerSource?: ResourceSource): Promise<void>;
+    configureStreaming(overlapSeconds?: number, windowSize?: number, streamingConfig?: keyof typeof MODES): void;
+    private chunkWaveform;
+    private resetState;
+    private expectedChunkLength;
+    private getStartingTokenIds;
+    private decodeChunk;
+    private handleOverlaps;
+    private trimLeft;
+    private trimRight;
+    private trimSequences;
+    private validateAndFixLastChunk;
+    private tokenIdsToText;
+    transcribe(waveform: number[], audioLanguage?: SpeechToTextLanguage): Promise<string>;
+    streamingTranscribe(streamAction: STREAMING_ACTION, waveform?: number[], audioLanguage?: SpeechToTextLanguage): Promise<string>;
+    encode(waveform: Float32Array): Promise<null>;
+    decode(seq: number[]): Promise<number>;
+}

package/lib/controllers/SpeechToTextController.js

@@ -0,0 +1,343 @@
+import { HAMMING_DIST_THRESHOLD, MODEL_CONFIGS, SECOND, MODES, NUM_TOKENS_TO_TRIM, STREAMING_ACTION, } from '../constants/sttDefaults';
+import { TokenizerModule } from '../modules/natural_language_processing/TokenizerModule';
+import { ResourceFetcher } from '../utils/ResourceFetcher';
+import { longCommonInfPref } from '../utils/stt';
+import { ETError, getError } from '../Error';
+import { Logger } from '../common/Logger';
+export class SpeechToTextController {
+    speechToTextNativeModule;
+    sequence = [];
+    isReady = false;
+    isGenerating = false;
+    tokenizerModule;
+    overlapSeconds;
+    windowSize;
+    chunks = [];
+    seqs = [];
+    prevSeq = [];
+    waveform = [];
+    numOfChunks = 0;
+    streaming = false;
+    // User callbacks
+    decodedTranscribeCallback;
+    modelDownloadProgressCallback;
+    isReadyCallback;
+    isGeneratingCallback;
+    onErrorCallback;
+    config;
+    constructor({ transcribeCallback, modelDownloadProgressCallback, isReadyCallback, isGeneratingCallback, onErrorCallback, overlapSeconds, windowSize, streamingConfig, }) {
+        this.tokenizerModule = new TokenizerModule();
+        this.decodedTranscribeCallback = async (seq) => transcribeCallback(await this.tokenIdsToText(seq));
+        this.modelDownloadProgressCallback = modelDownloadProgressCallback;
+        this.isReadyCallback = (isReady) => {
+            this.isReady = isReady;
+            isReadyCallback?.(isReady);
+        };
+        this.isGeneratingCallback = (isGenerating) => {
+            this.isGenerating = isGenerating;
+            isGeneratingCallback?.(isGenerating);
+        };
+        this.onErrorCallback = (error) => {
+            if (onErrorCallback) {
+                onErrorCallback(error ? new Error(getError(error)) : undefined);
+                return;
+            }
+            else {
+                throw new Error(getError(error));
+            }
+        };
+        this.configureStreaming(overlapSeconds, windowSize, streamingConfig || 'balanced');
+    }
+    async loadModel(modelName, encoderSource, decoderSource, tokenizerSource) {
+        this.onErrorCallback(undefined);
+        this.isReadyCallback(false);
+        this.config = MODEL_CONFIGS[modelName];
+        try {
+            await this.tokenizerModule.load(tokenizerSource || this.config.tokenizer.source);
+            const paths = await ResourceFetcher.fetch(this.modelDownloadProgressCallback, encoderSource || this.config.sources.encoder, decoderSource || this.config.sources.decoder);
+            if (paths === null || paths.length < 2) {
+                throw new Error('Download interrupted.');
+            }
+            [encoderSource, decoderSource] = paths;
+        }
+        catch (e) {
+            this.onErrorCallback(e);
+            return;
+        }
+        if (modelName === 'whisperMultilingual') {
+            // The underlying native class is instantiated based on the name of the model. There is no need to
+            // create a separate class for multilingual version of Whisper, since it is the same. We just need
+            // the distinction here, in TS, for start tokens and such. If we introduce
+            // more versions of Whisper, such as the small one, this should be refactored.
+            modelName = 'whisper';
+        }
+        try {
+            const nativeSpeechToText = await global.loadSpeechToText(encoderSource, decoderSource, modelName);
+            this.speechToTextNativeModule = nativeSpeechToText;
+            this.isReadyCallback(true);
+        }
+        catch (e) {
+            this.onErrorCallback(e);
+        }
+    }
+    configureStreaming(overlapSeconds, windowSize, streamingConfig) {
+        if (streamingConfig) {
+            this.windowSize = MODES[streamingConfig].windowSize * SECOND;
+            this.overlapSeconds = MODES[streamingConfig].overlapSeconds * SECOND;
+        }
+        if (streamingConfig && (windowSize || overlapSeconds)) {
+            Logger.warn(`windowSize and overlapSeconds overrides values from streamingConfig ${streamingConfig}.`);
+        }
+        this.windowSize = (windowSize || 0) * SECOND || this.windowSize;
+        this.overlapSeconds = (overlapSeconds || 0) * SECOND || this.overlapSeconds;
+        if (2 * this.overlapSeconds + this.windowSize >= 30 * SECOND) {
+            Logger.warn(`Invalid values for overlapSeconds and/or windowSize provided. Expected windowSize + 2 * overlapSeconds (== ${this.windowSize + 2 * this.overlapSeconds}) <= 30. Setting windowSize to ${30 * SECOND - 2 * this.overlapSeconds}.`);
+            this.windowSize = 30 * SECOND - 2 * this.overlapSeconds;
+        }
+    }
+    chunkWaveform() {
+        this.numOfChunks = Math.ceil(this.waveform.length / this.windowSize);
+        for (let i = 0; i < this.numOfChunks; i++) {
+            let chunk = [];
+            const left = Math.max(this.windowSize * i - this.overlapSeconds, 0);
+            const right = Math.min(this.windowSize * (i + 1) + this.overlapSeconds, this.waveform.length);
+            chunk = this.waveform.slice(left, right);
+            this.chunks.push(chunk);
+        }
+    }
+    resetState() {
+        this.sequence = [];
+        this.seqs = [];
+        this.waveform = [];
+        this.prevSeq = [];
+        this.chunks = [];
+        this.decodedTranscribeCallback([]);
+        this.onErrorCallback(undefined);
+    }
+    expectedChunkLength() {
+        //only first chunk can be of shorter length, for first chunk there are no seqs decoded
+        return this.seqs.length
+            ? this.windowSize + 2 * this.overlapSeconds
+            : this.windowSize + this.overlapSeconds;
+    }
+    async getStartingTokenIds(audioLanguage) {
+        // We need different starting token ids based on the multilingualism of the model.
+        // The eng version only needs BOS token, while the multilingual one needs:
+        // [BOS, LANG, TRANSCRIBE]. Optionally we should also set notimestamps token, as timestamps
+        // is not yet supported.
+        if (!audioLanguage) {
+            return [this.config.tokenizer.bos];
+        }
+        // FIXME: I should use .getTokenId for the BOS as well, should remove it from config
+        const langTokenId = await this.tokenizerModule.tokenToId(`<|${audioLanguage}|>`);
+        const transcribeTokenId = await this.tokenizerModule.tokenToId('<|transcribe|>');
+        const noTimestampsTokenId = await this.tokenizerModule.tokenToId('<|notimestamps|>');
+        const startingTokenIds = [
+            this.config.tokenizer.bos,
+            langTokenId,
+            transcribeTokenId,
+            noTimestampsTokenId,
+        ];
+        return startingTokenIds;
+    }
+    async decodeChunk(chunk, audioLanguage) {
+        const seq = await this.getStartingTokenIds(audioLanguage);
+        let prevSeqTokenIdx = 0;
+        this.prevSeq = this.sequence.slice();
+        try {
+            await this.encode(new Float32Array(chunk));
+        }
+        catch (error) {
+            this.onErrorCallback(new Error(getError(error) + ' encoding error'));
+            return [];
+        }
+        let lastToken = seq.at(-1);
+        while (lastToken !== this.config.tokenizer.eos) {
+            try {
+                lastToken = await this.decode(seq);
+            }
+            catch (error) {
+                this.onErrorCallback(new Error(getError(error) + ' decoding error'));
+                return [...seq, this.config.tokenizer.eos];
+            }
+            seq.push(lastToken);
+            if (this.seqs.length > 0 &&
+                seq.length < this.seqs.at(-1).length &&
+                seq.length % 3 !== 0) {
+                this.prevSeq.push(this.seqs.at(-1)[prevSeqTokenIdx++]);
+                this.decodedTranscribeCallback(this.prevSeq);
+            }
+        }
+        return seq;
+    }
+    async handleOverlaps(seqs) {
+        const maxInd = longCommonInfPref(seqs.at(-2), seqs.at(-1), HAMMING_DIST_THRESHOLD);
+        this.sequence = [...this.sequence, ...seqs.at(-2).slice(0, maxInd)];
+        this.decodedTranscribeCallback(this.sequence);
+        return this.sequence.slice();
+    }
+    trimLeft(numOfTokensToTrim) {
+        const idx = this.seqs.length - 1;
+        if (this.seqs[idx][0] === this.config.tokenizer.bos) {
+            this.seqs[idx] = this.seqs[idx].slice(numOfTokensToTrim);
+        }
+    }
+    trimRight(numOfTokensToTrim) {
+        const idx = this.seqs.length - 2;
+        if (this.seqs[idx].at(-1) === this.config.tokenizer.eos) {
+            this.seqs[idx] = this.seqs[idx].slice(0, -numOfTokensToTrim);
+        }
+    }
+    // since we are calling this every time (except first) after a new seq is pushed to this.seqs
+    // we can only trim left the last seq and trim right the second to last seq
+    async trimSequences(audioLanguage) {
+        const numSpecialTokens = (await this.getStartingTokenIds(audioLanguage))
+            .length;
+        this.trimLeft(numSpecialTokens + NUM_TOKENS_TO_TRIM);
+        this.trimRight(numSpecialTokens + NUM_TOKENS_TO_TRIM);
+    }
+    // if last chunk is too short combine it with second to last to improve quality
+    validateAndFixLastChunk() {
+        if (this.chunks.length < 2)
+            return;
+        const lastChunkLength = this.chunks.at(-1).length / SECOND;
+        const secondToLastChunkLength = this.chunks.at(-2).length / SECOND;
+        if (lastChunkLength < 5 && secondToLastChunkLength + lastChunkLength < 30) {
+            this.chunks[this.chunks.length - 2] = [
+                ...this.chunks.at(-2).slice(0, -this.overlapSeconds * 2),
+                ...this.chunks.at(-1),
+            ];
+            this.chunks = this.chunks.slice(0, -1);
+        }
+    }
+    async tokenIdsToText(tokenIds) {
+        try {
+            return await this.tokenizerModule.decode(tokenIds, true);
+        }
+        catch (e) {
+            this.onErrorCallback(new Error(`An error has occurred when decoding the token ids: ${e}`));
+            return '';
+        }
+    }
+    async transcribe(waveform, audioLanguage) {
+        try {
+            if (!this.isReady)
+                throw Error(getError(ETError.ModuleNotLoaded));
+            if (this.isGenerating || this.streaming)
+                throw Error(getError(ETError.ModelGenerating));
+            if (!!audioLanguage !== this.config.isMultilingual)
+                throw new Error(getError(ETError.MultilingualConfiguration));
+        }
+        catch (e) {
+            this.onErrorCallback(e);
+            return '';
+        }
+        // Making sure that the error is not set when we get there
+        this.isGeneratingCallback(true);
+        this.resetState();
+        this.waveform = waveform;
+        this.chunkWaveform();
+        this.validateAndFixLastChunk();
+        for (let chunkId = 0; chunkId < this.chunks.length; chunkId++) {
+            const seq = await this.decodeChunk(this.chunks.at(chunkId), audioLanguage);
+            // whole audio is inside one chunk, no processing required
+            if (this.chunks.length === 1) {
+                this.sequence = seq;
+                this.decodedTranscribeCallback(seq);
+                break;
+            }
+            this.seqs.push(seq);
+            if (this.seqs.length < 2)
+                continue;
+            // Remove starting tokenIds and some additional ones
+            await this.trimSequences(audioLanguage);
+            this.prevSeq = await this.handleOverlaps(this.seqs);
+            // last sequence processed
+            // overlaps are already handled, so just append the last seq
+            if (this.seqs.length === this.chunks.length) {
+                this.sequence = [...this.sequence, ...this.seqs.at(-1)];
+                this.decodedTranscribeCallback(this.sequence);
+                this.prevSeq = this.sequence;
+            }
+        }
+        const decodedText = await this.tokenIdsToText(this.sequence);
+        this.isGeneratingCallback(false);
+        return decodedText;
+    }
+    async streamingTranscribe(streamAction, waveform, audioLanguage) {
+        try {
+            if (!this.isReady)
+                throw Error(getError(ETError.ModuleNotLoaded));
+            if (!!audioLanguage !== this.config.isMultilingual)
+                throw new Error(getError(ETError.MultilingualConfiguration));
+            if (streamAction === STREAMING_ACTION.START &&
+                !this.streaming &&
+                this.isGenerating)
+                throw Error(getError(ETError.ModelGenerating));
+            if (streamAction === STREAMING_ACTION.START && this.streaming)
+                throw Error(getError(ETError.ModelGenerating));
+            if (streamAction === STREAMING_ACTION.DATA && !this.streaming)
+                throw Error(getError(ETError.StreamingNotStarted));
+            if (streamAction === STREAMING_ACTION.STOP && !this.streaming)
+                throw Error(getError(ETError.StreamingNotStarted));
+            if (streamAction === STREAMING_ACTION.DATA && !waveform)
+                throw new Error(getError(ETError.MissingDataChunk));
+        }
+        catch (e) {
+            this.onErrorCallback(e);
+            return '';
+        }
+        if (streamAction === STREAMING_ACTION.START) {
+            this.resetState();
+            this.streaming = true;
+            this.isGeneratingCallback(true);
+        }
+        this.waveform = [...this.waveform, ...(waveform || [])];
+        // while buffer has at least required size get chunk and decode
+        while (this.waveform.length >= this.expectedChunkLength()) {
+            const chunk = this.waveform.slice(0, this.windowSize +
+                this.overlapSeconds * (1 + Number(this.seqs.length > 0)));
+            this.chunks = [chunk]; //save last chunk for STREAMING_ACTION.STOP
+            this.waveform = this.waveform.slice(this.windowSize - this.overlapSeconds * Number(this.seqs.length === 0));
+            const seq = await this.decodeChunk(chunk, audioLanguage);
+            this.seqs.push(seq);
+            if (this.seqs.length < 2)
+                continue;
+            await this.trimSequences(audioLanguage);
+            await this.handleOverlaps(this.seqs);
+        }
+        // got final package, process all remaining waveform data
+        // since we run the loop above the waveform has at most one chunk in it
+        if (streamAction === STREAMING_ACTION.STOP) {
+            // pad remaining waveform data with previous chunk to this.windowSize + 2 * this.overlapSeconds
+            const chunk = this.chunks.length
+                ? [
+                    ...this.chunks[0].slice(0, this.windowSize),
+                    ...this.waveform,
+                ].slice(-this.windowSize - 2 * this.overlapSeconds)
+                : this.waveform;
+            this.waveform = [];
+            const seq = await this.decodeChunk(chunk, audioLanguage);
+            this.seqs.push(seq);
+            if (this.seqs.length === 1) {
+                this.sequence = this.seqs[0];
+            }
+            else {
+                await this.trimSequences(audioLanguage);
+                await this.handleOverlaps(this.seqs);
+                this.sequence = [...this.sequence, ...this.seqs.at(-1)];
+            }
+            this.decodedTranscribeCallback(this.sequence);
+            this.isGeneratingCallback(false);
+            this.streaming = false;
+        }
+        const decodedText = await this.tokenIdsToText(this.sequence);
+        return decodedText;
+    }
+    async encode(waveform) {
+        return await this.speechToTextNativeModule.encode(waveform);
+    }
+    async decode(seq) {
+        return await this.speechToTextNativeModule.decode(seq);
+    }
+}