react-native-executorch 0.5.1 → 0.5.3

package/lib/controllers/VerticalOCRController.js

@@ -0,0 +1,70 @@
+import { symbols } from '../constants/ocr/symbols';
+import { ETError, getError } from '../Error';
+import { VerticalOCRNativeModule } from '../native/RnExecutorchModules';
+import { ResourceFetcher } from '../utils/ResourceFetcher';
+export class VerticalOCRController {
+    ocrNativeModule;
+    isReady = false;
+    isGenerating = false;
+    error = null;
+    modelDownloadProgressCallback;
+    isReadyCallback;
+    isGeneratingCallback;
+    errorCallback;
+    constructor({ modelDownloadProgressCallback = (_downloadProgress) => { }, isReadyCallback = (_isReady) => { }, isGeneratingCallback = (_isGenerating) => { }, errorCallback = (_error) => { }, }) {
+        this.ocrNativeModule = VerticalOCRNativeModule;
+        this.modelDownloadProgressCallback = modelDownloadProgressCallback;
+        this.isReadyCallback = isReadyCallback;
+        this.isGeneratingCallback = isGeneratingCallback;
+        this.errorCallback = errorCallback;
+    }
+    loadModel = async (detectorSources, recognizerSources, language, independentCharacters) => {
+        try {
+            if (Object.keys(detectorSources).length !== 2 ||
+                Object.keys(recognizerSources).length !== 2)
+                return;
+            if (!symbols[language]) {
+                throw new Error(getError(ETError.LanguageNotSupported));
+            }
+            this.isReady = false;
+            this.isReadyCallback(this.isReady);
+            const paths = await ResourceFetcher.fetch(this.modelDownloadProgressCallback, detectorSources.detectorLarge, detectorSources.detectorNarrow, independentCharacters
+                ? recognizerSources.recognizerSmall
+                : recognizerSources.recognizerLarge);
+            if (paths === null || paths.length < 3) {
+                throw new Error('Download interrupted');
+            }
+            await this.ocrNativeModule.loadModule(paths[0], paths[1], paths[2], symbols[language], independentCharacters);
+            this.isReady = true;
+            this.isReadyCallback(this.isReady);
+        }
+        catch (e) {
+            if (this.errorCallback) {
+                this.errorCallback(getError(e));
+            }
+            else {
+                throw new Error(getError(e));
+            }
+        }
+    };
+    forward = async (input) => {
+        if (!this.isReady) {
+            throw new Error(getError(ETError.ModuleNotLoaded));
+        }
+        if (this.isGenerating) {
+            throw new Error(getError(ETError.ModelGenerating));
+        }
+        try {
+            this.isGenerating = true;
+            this.isGeneratingCallback(this.isGenerating);
+            return await this.ocrNativeModule.forward(input);
+        }
+        catch (e) {
+            throw new Error(getError(e));
+        }
+        finally {
+            this.isGenerating = false;
+            this.isGeneratingCallback(this.isGenerating);
+        }
+    };
+}

package/lib/hooks/computer_vision/useClassification.d.ts

@@ -0,0 +1,15 @@
+import { ResourceSource } from '../../types/common';
+interface Props {
+    model: {
+        modelSource: ResourceSource;
+    };
+    preventLoad?: boolean;
+}
+export declare const useClassification: ({ model, preventLoad }: Props) => {
+    error: string | null;
+    isReady: boolean;
+    isGenerating: boolean;
+    downloadProgress: number;
+    forward: (imageSource: string) => Promise<any>;
+};
+export {};

package/lib/hooks/computer_vision/useClassification.js

@@ -0,0 +1,7 @@
+import { useNonStaticModule } from '../useNonStaticModule';
+import { ClassificationModule } from '../../modules/computer_vision/ClassificationModule';
+export const useClassification = ({ model, preventLoad = false }) => useNonStaticModule({
+    module: ClassificationModule,
+    model,
+    preventLoad: preventLoad,
+});

package/lib/hooks/computer_vision/useImageEmbeddings.d.ts

@@ -0,0 +1,15 @@
+import { ResourceSource } from '../../types/common';
+interface Props {
+    model: {
+        modelSource: ResourceSource;
+    };
+    preventLoad?: boolean;
+}
+export declare const useImageEmbeddings: ({ model, preventLoad }: Props) => {
+    error: string | null;
+    isReady: boolean;
+    isGenerating: boolean;
+    downloadProgress: number;
+    forward: (imageSource: string) => Promise<Float32Array<ArrayBufferLike>>;
+};
+export {};

package/lib/hooks/computer_vision/useImageEmbeddings.js

@@ -0,0 +1,7 @@
+import { ImageEmbeddingsModule } from '../../modules/computer_vision/ImageEmbeddingsModule';
+import { useNonStaticModule } from '../useNonStaticModule';
+export const useImageEmbeddings = ({ model, preventLoad = false }) => useNonStaticModule({
+    module: ImageEmbeddingsModule,
+    model,
+    preventLoad,
+});

package/lib/hooks/computer_vision/useImageSegmentation.d.ts

@@ -0,0 +1,38 @@
+import { ResourceSource } from '../../types/common';
+interface Props {
+    model: {
+        modelSource: ResourceSource;
+    };
+    preventLoad?: boolean;
+}
+export declare const useImageSegmentation: ({ model, preventLoad }: Props) => {
+    error: string | null;
+    isReady: boolean;
+    isGenerating: boolean;
+    downloadProgress: number;
+    forward: (imageSource: string, classesOfInterest?: import("../..").DeeplabLabel[] | undefined, resize?: boolean | undefined) => Promise<{
+        0?: number[] | undefined;
+        1?: number[] | undefined;
+        2?: number[] | undefined;
+        3?: number[] | undefined;
+        4?: number[] | undefined;
+        5?: number[] | undefined;
+        6?: number[] | undefined;
+        7?: number[] | undefined;
+        8?: number[] | undefined;
+        9?: number[] | undefined;
+        10?: number[] | undefined;
+        11?: number[] | undefined;
+        12?: number[] | undefined;
+        13?: number[] | undefined;
+        14?: number[] | undefined;
+        15?: number[] | undefined;
+        16?: number[] | undefined;
+        17?: number[] | undefined;
+        18?: number[] | undefined;
+        19?: number[] | undefined;
+        20?: number[] | undefined;
+        21?: number[] | undefined;
+    }>;
+};
+export {};

package/lib/hooks/computer_vision/useImageSegmentation.js

@@ -0,0 +1,7 @@
+import { useNonStaticModule } from '../useNonStaticModule';
+import { ImageSegmentationModule } from '../../modules/computer_vision/ImageSegmentationModule';
+export const useImageSegmentation = ({ model, preventLoad = false }) => useNonStaticModule({
+    module: ImageSegmentationModule,
+    model,
+    preventLoad,
+});

package/lib/hooks/computer_vision/useOCR.d.ts

@@ -0,0 +1,20 @@
+import { ResourceSource } from '../../types/common';
+import { OCRDetection, OCRLanguage } from '../../types/ocr';
+interface OCRModule {
+    error: string | null;
+    isReady: boolean;
+    isGenerating: boolean;
+    forward: (input: string) => Promise<OCRDetection[]>;
+    downloadProgress: number;
+}
+export declare const useOCR: ({ model, preventLoad, }: {
+    model: {
+        detectorSource: ResourceSource;
+        recognizerLarge: ResourceSource;
+        recognizerMedium: ResourceSource;
+        recognizerSmall: ResourceSource;
+        language: OCRLanguage;
+    };
+    preventLoad?: boolean;
+}) => OCRModule;
+export {};

package/lib/hooks/computer_vision/useOCR.js

@@ -0,0 +1,41 @@
+import { useEffect, useMemo, useState } from 'react';
+import { OCRController } from '../../controllers/OCRController';
+export const useOCR = ({ model, preventLoad = false, }) => {
+    const [error, setError] = useState(null);
+    const [isReady, setIsReady] = useState(false);
+    const [isGenerating, setIsGenerating] = useState(false);
+    const [downloadProgress, setDownloadProgress] = useState(0);
+    const controllerInstance = useMemo(() => new OCRController({
+        modelDownloadProgressCallback: setDownloadProgress,
+        isReadyCallback: setIsReady,
+        isGeneratingCallback: setIsGenerating,
+        errorCallback: setError,
+    }), []);
+    useEffect(() => {
+        const loadModel = async () => {
+            await controllerInstance.loadModel(model.detectorSource, {
+                recognizerLarge: model.recognizerLarge,
+                recognizerMedium: model.recognizerMedium,
+                recognizerSmall: model.recognizerSmall,
+            }, model.language);
+        };
+        if (!preventLoad) {
+            loadModel();
+        }
+    }, [
+        controllerInstance,
+        model.detectorSource,
+        model.recognizerLarge,
+        model.recognizerMedium,
+        model.recognizerSmall,
+        model.language,
+        preventLoad,
+    ]);
+    return {
+        error,
+        isReady,
+        isGenerating,
+        forward: controllerInstance.forward,
+        downloadProgress,
+    };
+};

package/lib/hooks/computer_vision/useObjectDetection.d.ts

@@ -0,0 +1,15 @@
+import { ResourceSource } from '../../types/common';
+interface Props {
+    model: {
+        modelSource: ResourceSource;
+    };
+    preventLoad?: boolean;
+}
+export declare const useObjectDetection: ({ model, preventLoad }: Props) => {
+    error: string | null;
+    isReady: boolean;
+    isGenerating: boolean;
+    downloadProgress: number;
+    forward: (imageSource: string, detectionThreshold?: number | undefined) => Promise<import("../..").Detection[]>;
+};
+export {};

package/lib/hooks/computer_vision/useObjectDetection.js

@@ -0,0 +1,7 @@
+import { useNonStaticModule } from '../useNonStaticModule';
+import { ObjectDetectionModule } from '../../modules/computer_vision/ObjectDetectionModule';
+export const useObjectDetection = ({ model, preventLoad = false }) => useNonStaticModule({
+    module: ObjectDetectionModule,
+    model,
+    preventLoad: preventLoad,
+});

package/lib/hooks/computer_vision/useStyleTransfer.d.ts

@@ -0,0 +1,15 @@
+import { ResourceSource } from '../../types/common';
+interface Props {
+    model: {
+        modelSource: ResourceSource;
+    };
+    preventLoad?: boolean;
+}
+export declare const useStyleTransfer: ({ model, preventLoad }: Props) => {
+    error: string | null;
+    isReady: boolean;
+    isGenerating: boolean;
+    downloadProgress: number;
+    forward: (imageSource: string) => Promise<string>;
+};
+export {};

package/lib/hooks/computer_vision/useStyleTransfer.js

@@ -0,0 +1,7 @@
+import { useNonStaticModule } from '../useNonStaticModule';
+import { StyleTransferModule } from '../../modules/computer_vision/StyleTransferModule';
+export const useStyleTransfer = ({ model, preventLoad = false }) => useNonStaticModule({
+    module: StyleTransferModule,
+    model,
+    preventLoad: preventLoad,
+});

package/lib/hooks/computer_vision/useVerticalOCR.d.ts

@@ -0,0 +1,21 @@
+import { ResourceSource } from '../../types/common';
+import { OCRDetection, OCRLanguage } from '../../types/ocr';
+interface OCRModule {
+    error: string | null;
+    isReady: boolean;
+    isGenerating: boolean;
+    forward: (input: string) => Promise<OCRDetection[]>;
+    downloadProgress: number;
+}
+export declare const useVerticalOCR: ({ model, independentCharacters, preventLoad, }: {
+    model: {
+        detectorLarge: ResourceSource;
+        detectorNarrow: ResourceSource;
+        recognizerLarge: ResourceSource;
+        recognizerSmall: ResourceSource;
+        language: OCRLanguage;
+    };
+    independentCharacters?: boolean;
+    preventLoad?: boolean;
+}) => OCRModule;
+export {};

package/lib/hooks/computer_vision/useVerticalOCR.js

@@ -0,0 +1,43 @@
+import { useEffect, useMemo, useState } from 'react';
+import { VerticalOCRController } from '../../controllers/VerticalOCRController';
+export const useVerticalOCR = ({ model, independentCharacters = false, preventLoad = false, }) => {
+    const [error, setError] = useState(null);
+    const [isReady, setIsReady] = useState(false);
+    const [isGenerating, setIsGenerating] = useState(false);
+    const [downloadProgress, setDownloadProgress] = useState(0);
+    const controllerInstance = useMemo(() => new VerticalOCRController({
+        modelDownloadProgressCallback: setDownloadProgress,
+        isReadyCallback: setIsReady,
+        isGeneratingCallback: setIsGenerating,
+        errorCallback: setError,
+    }), []);
+    useEffect(() => {
+        if (preventLoad)
+            return;
+        (async () => {
+            await controllerInstance.loadModel({
+                detectorLarge: model.detectorLarge,
+                detectorNarrow: model.detectorNarrow,
+            }, {
+                recognizerLarge: model.recognizerLarge,
+                recognizerSmall: model.recognizerSmall,
+            }, model.language, independentCharacters);
+        })();
+    }, [
+        controllerInstance,
+        model.detectorLarge,
+        model.detectorNarrow,
+        model.recognizerLarge,
+        model.recognizerSmall,
+        model.language,
+        independentCharacters,
+        preventLoad,
+    ]);
+    return {
+        error,
+        isReady,
+        isGenerating,
+        forward: controllerInstance.forward,
+        downloadProgress,
+    };
+};

package/lib/hooks/general/useExecutorchModule.d.ts

@@ -0,0 +1,13 @@
+import { ResourceSource } from '../../types/common';
+interface Props {
+    modelSource: ResourceSource;
+    preventLoad?: boolean;
+}
+export declare const useExecutorchModule: ({ modelSource, preventLoad, }: Props) => {
+    error: string | null;
+    isReady: boolean;
+    isGenerating: boolean;
+    downloadProgress: number;
+    forward: (inputTensor: import("../../types/common").TensorPtr[]) => Promise<import("../../types/common").TensorPtr[]>;
+};
+export {};

package/lib/hooks/general/useExecutorchModule.js

@@ -0,0 +1,7 @@
+import { ExecutorchModule } from '../../modules/general/ExecutorchModule';
+import { useNonStaticModule } from '../useNonStaticModule';
+export const useExecutorchModule = ({ modelSource, preventLoad = false, }) => useNonStaticModule({
+    module: ExecutorchModule,
+    model: { modelSource },
+    preventLoad,
+});

package/lib/hooks/natural_language_processing/useLLM.d.ts

@@ -0,0 +1,10 @@
+import { ResourceSource } from '../../types/common';
+import { LLMType } from '../../types/llm';
+export declare const useLLM: ({ model, preventLoad, }: {
+    model: {
+        modelSource: ResourceSource;
+        tokenizerSource: ResourceSource;
+        tokenizerConfigSource: ResourceSource;
+    };
+    preventLoad?: boolean;
+}) => LLMType;

package/lib/hooks/natural_language_processing/useLLM.js

@@ -0,0 +1,78 @@
+import { useCallback, useEffect, useMemo, useState } from 'react';
+import { LLMController } from '../../controllers/LLMController';
+/*
+Hook version of LLMModule
+*/
+export const useLLM = ({ model, preventLoad = false, }) => {
+    const [token, setToken] = useState('');
+    const [response, setResponse] = useState('');
+    const [messageHistory, setMessageHistory] = useState([]);
+    const [isReady, setIsReady] = useState(false);
+    const [isGenerating, setIsGenerating] = useState(false);
+    const [downloadProgress, setDownloadProgress] = useState(0);
+    const [error, setError] = useState(null);
+    const tokenCallback = useCallback((newToken) => {
+        setToken(newToken);
+        setResponse((prevResponse) => prevResponse + newToken);
+    }, []);
+    const controllerInstance = useMemo(() => new LLMController({
+        tokenCallback: tokenCallback,
+        messageHistoryCallback: setMessageHistory,
+        isReadyCallback: setIsReady,
+        isGeneratingCallback: setIsGenerating,
+    }), [tokenCallback]);
+    useEffect(() => {
+        setDownloadProgress(0);
+        setError(null);
+        if (preventLoad)
+            return;
+        (async () => {
+            try {
+                await controllerInstance.load({
+                    modelSource: model.modelSource,
+                    tokenizerSource: model.tokenizerSource,
+                    tokenizerConfigSource: model.tokenizerConfigSource,
+                    onDownloadProgressCallback: setDownloadProgress,
+                });
+            }
+            catch (e) {
+                setError(e);
+            }
+        })();
+        return () => {
+            controllerInstance.delete();
+        };
+    }, [
+        controllerInstance,
+        model.modelSource,
+        model.tokenizerSource,
+        model.tokenizerConfigSource,
+        preventLoad,
+    ]);
+    // memoization of returned functions
+    const configure = useCallback(({ chatConfig, toolsConfig, }) => controllerInstance.configure({ chatConfig, toolsConfig }), [controllerInstance]);
+    const generate = useCallback((messages, tools) => {
+        setResponse('');
+        return controllerInstance.generate(messages, tools);
+    }, [controllerInstance]);
+    const sendMessage = useCallback((message) => {
+        setResponse('');
+        return controllerInstance.sendMessage(message);
+    }, [controllerInstance]);
+    const deleteMessage = useCallback((index) => controllerInstance.deleteMessage(index), [controllerInstance]);
+    const interrupt = useCallback(() => controllerInstance.interrupt(), [controllerInstance]);
+    return {
+        messageHistory,
+        response,
+        token,
+        isReady,
+        isGenerating,
+        downloadProgress,
+        error,
+        configure: configure,
+        generate: generate,
+        sendMessage: sendMessage,
+        deleteMessage: deleteMessage,
+        interrupt: interrupt,
+    };
+};

package/lib/hooks/natural_language_processing/useSpeechToText.d.ts

@@ -0,0 +1,27 @@
+import { SpeechToTextController } from '../../controllers/SpeechToTextController';
+import { ResourceSource } from '../../types/common';
+import { STREAMING_ACTION } from '../../constants/sttDefaults';
+import { AvailableModels, SpeechToTextLanguage } from '../../types/stt';
+interface SpeechToTextModule {
+    isReady: boolean;
+    isGenerating: boolean;
+    sequence: string;
+    downloadProgress: number;
+    configureStreaming: SpeechToTextController['configureStreaming'];
+    error: Error | undefined;
+    transcribe: (input: number[], audioLanguage?: SpeechToTextLanguage) => ReturnType<SpeechToTextController['transcribe']>;
+    streamingTranscribe: (streamAction: STREAMING_ACTION, input?: number[], audioLanguage?: SpeechToTextLanguage) => ReturnType<SpeechToTextController['streamingTranscribe']>;
+}
+export declare const useSpeechToText: ({ model, overlapSeconds, windowSize, streamingConfig, preventLoad, }: {
+    model: {
+        modelName: AvailableModels;
+        encoderSource: ResourceSource;
+        decoderSource: ResourceSource;
+        tokenizerSource: ResourceSource;
+    };
+    overlapSeconds?: ConstructorParameters<typeof SpeechToTextController>["0"]["overlapSeconds"];
+    windowSize?: ConstructorParameters<typeof SpeechToTextController>["0"]["windowSize"];
+    streamingConfig?: ConstructorParameters<typeof SpeechToTextController>["0"]["streamingConfig"];
+    preventLoad?: boolean;
+}) => SpeechToTextModule;
+export {};

package/lib/hooks/natural_language_processing/useSpeechToText.js

@@ -0,0 +1,49 @@
+import { useEffect, useMemo, useState } from 'react';
+import { SpeechToTextController } from '../../controllers/SpeechToTextController';
+export const useSpeechToText = ({ model, overlapSeconds, windowSize, streamingConfig, preventLoad = false, }) => {
+    const [sequence, setSequence] = useState('');
+    const [isReady, setIsReady] = useState(false);
+    const [downloadProgress, setDownloadProgress] = useState(0);
+    const [isGenerating, setIsGenerating] = useState(false);
+    const [error, setError] = useState();
+    const controllerInstance = useMemo(() => new SpeechToTextController({
+        transcribeCallback: setSequence,
+        isReadyCallback: setIsReady,
+        isGeneratingCallback: setIsGenerating,
+        onErrorCallback: setError,
+    }), []);
+    useEffect(() => {
+        controllerInstance.configureStreaming(overlapSeconds, windowSize, streamingConfig);
+    }, [controllerInstance, overlapSeconds, windowSize, streamingConfig]);
+    useEffect(() => {
+        const loadModel = async () => {
+            await controllerInstance.load({
+                modelName: model.modelName,
+                encoderSource: model.encoderSource,
+                decoderSource: model.decoderSource,
+                tokenizerSource: model.tokenizerSource,
+                onDownloadProgressCallback: setDownloadProgress,
+            });
+        };
+        if (!preventLoad) {
+            loadModel();
+        }
+    }, [
+        controllerInstance,
+        model.modelName,
+        model.encoderSource,
+        model.decoderSource,
+        model.tokenizerSource,
+        preventLoad,
+    ]);
+    return {
+        isReady,
+        isGenerating,
+        downloadProgress,
+        configureStreaming: controllerInstance.configureStreaming,
+        sequence,
+        error,
+        transcribe: (waveform, audioLanguage) => controllerInstance.transcribe(waveform, audioLanguage),
+        streamingTranscribe: (streamAction, waveform, audioLanguage) => controllerInstance.streamingTranscribe(streamAction, waveform, audioLanguage),
+    };
+};

package/lib/hooks/natural_language_processing/useTextEmbeddings.d.ts

@@ -0,0 +1,16 @@
+import { ResourceSource } from '../../types/common';
+interface Props {
+    model: {
+        modelSource: ResourceSource;
+        tokenizerSource: ResourceSource;
+    };
+    preventLoad?: boolean;
+}
+export declare const useTextEmbeddings: ({ model, preventLoad }: Props) => {
+    error: string | null;
+    isReady: boolean;
+    isGenerating: boolean;
+    downloadProgress: number;
+    forward: (input: string) => Promise<Float32Array<ArrayBufferLike>>;
+};
+export {};

package/lib/hooks/natural_language_processing/useTextEmbeddings.js

@@ -0,0 +1,7 @@
+import { TextEmbeddingsModule } from '../../modules/natural_language_processing/TextEmbeddingsModule';
+import { useNonStaticModule } from '../useNonStaticModule';
+export const useTextEmbeddings = ({ model, preventLoad = false }) => useNonStaticModule({
+    module: TextEmbeddingsModule,
+    model,
+    preventLoad,
+});

package/lib/hooks/natural_language_processing/useTokenizer.d.ts

@@ -0,0 +1,17 @@
+import { ResourceSource } from '../../types/common';
+export declare const useTokenizer: ({ tokenizer, preventLoad, }: {
+    tokenizer: {
+        tokenizerSource: ResourceSource;
+    };
+    preventLoad?: boolean;
+}) => {
+    error: string | null;
+    isReady: boolean;
+    isGenerating: boolean;
+    downloadProgress: number;
+    decode: (tokens: number[], skipSpecialTokens?: boolean | undefined) => Promise<Promise<any>>;
+    encode: (s: string) => Promise<Promise<any>>;
+    getVocabSize: () => Promise<Promise<number>>;
+    idToToken: (tokenId: number) => Promise<Promise<string>>;
+    tokenToId: (token: string) => Promise<Promise<number>>;
+};

package/lib/hooks/natural_language_processing/useTokenizer.js

@@ -0,0 +1,52 @@
+import { useEffect, useMemo, useState } from 'react';
+import { TokenizerModule } from '../../modules/natural_language_processing/TokenizerModule';
+import { ETError, getError } from '../../Error';
+export const useTokenizer = ({ tokenizer, preventLoad = false, }) => {
+    const [error, setError] = useState(null);
+    const [isReady, setIsReady] = useState(false);
+    const [isGenerating, setIsGenerating] = useState(false);
+    const [downloadProgress, setDownloadProgress] = useState(0);
+    const _tokenizer = useMemo(() => new TokenizerModule(), []);
+    useEffect(() => {
+        if (preventLoad)
+            return;
+        (async () => {
+            setDownloadProgress(0);
+            setError(null);
+            try {
+                setIsReady(false);
+                await _tokenizer.load({ tokenizerSource: tokenizer.tokenizerSource }, setDownloadProgress);
+                setIsReady(true);
+            }
+            catch (err) {
+                setError(err.message);
+            }
+        })();
+    }, [_tokenizer, tokenizer.tokenizerSource, preventLoad]);
+    const stateWrapper = (fn) => {
+        return (...args) => {
+            if (!isReady)
+                throw new Error(getError(ETError.ModuleNotLoaded));
+            if (isGenerating)
+                throw new Error(getError(ETError.ModelGenerating));
+            try {
+                setIsGenerating(true);
+                return fn.apply(_tokenizer, args);
+            }
+            finally {
+                setIsGenerating(false);
+            }
+        };
+    };
+    return {
+        error,
+        isReady,
+        isGenerating,
+        downloadProgress,
+        decode: stateWrapper(TokenizerModule.prototype.decode),
+        encode: stateWrapper(TokenizerModule.prototype.encode),
+        getVocabSize: stateWrapper(TokenizerModule.prototype.getVocabSize),
+        idToToken: stateWrapper(TokenizerModule.prototype.idToToken),
+        tokenToId: stateWrapper(TokenizerModule.prototype.tokenToId),
+    };
+};