react-native-executorch 0.5.1 → 0.5.2

package/lib/controllers/SpeechToTextController.js

@@ -0,0 +1,349 @@
+import { HAMMING_DIST_THRESHOLD, MODEL_CONFIGS, SECOND, MODES, NUM_TOKENS_TO_TRIM, STREAMING_ACTION, } from '../constants/sttDefaults';
+import { AvailableModels } from '../types/stt';
+import { TokenizerModule } from '../modules/natural_language_processing/TokenizerModule';
+import { ResourceFetcher } from '../utils/ResourceFetcher';
+import { longCommonInfPref } from '../utils/stt';
+import { ETError, getError } from '../Error';
+import { Logger } from '../common/Logger';
+export class SpeechToTextController {
+    speechToTextNativeModule;
+    sequence = [];
+    isReady = false;
+    isGenerating = false;
+    tokenizerModule;
+    overlapSeconds;
+    windowSize;
+    chunks = [];
+    seqs = [];
+    prevSeq = [];
+    waveform = [];
+    numOfChunks = 0;
+    streaming = false;
+    // User callbacks
+    decodedTranscribeCallback;
+    isReadyCallback;
+    isGeneratingCallback;
+    onErrorCallback;
+    config;
+    constructor({ transcribeCallback, isReadyCallback, isGeneratingCallback, onErrorCallback, overlapSeconds, windowSize, streamingConfig, }) {
+        this.tokenizerModule = new TokenizerModule();
+        this.decodedTranscribeCallback = async (seq) => transcribeCallback(await this.tokenIdsToText(seq));
+        this.isReadyCallback = (isReady) => {
+            this.isReady = isReady;
+            isReadyCallback?.(isReady);
+        };
+        this.isGeneratingCallback = (isGenerating) => {
+            this.isGenerating = isGenerating;
+            isGeneratingCallback?.(isGenerating);
+        };
+        this.onErrorCallback = (error) => {
+            if (onErrorCallback) {
+                onErrorCallback(error ? new Error(getError(error)) : undefined);
+                return;
+            }
+            else {
+                throw new Error(getError(error));
+            }
+        };
+        this.configureStreaming(overlapSeconds, windowSize, streamingConfig || 'balanced');
+    }
+    async load({ modelName, encoderSource, decoderSource, tokenizerSource, onDownloadProgressCallback, }) {
+        this.onErrorCallback(undefined);
+        this.isReadyCallback(false);
+        this.config = MODEL_CONFIGS[modelName];
+        try {
+            const tokenizerLoadPromise = this.tokenizerModule.load({
+                tokenizerSource: tokenizerSource || this.config.tokenizer.source,
+            });
+            const pathsPromise = ResourceFetcher.fetch(onDownloadProgressCallback, encoderSource || this.config.sources.encoder, decoderSource || this.config.sources.decoder);
+            const [_, encoderDecoderResults] = await Promise.all([
+                tokenizerLoadPromise,
+                pathsPromise,
+            ]);
+            encoderSource = encoderDecoderResults?.[0];
+            decoderSource = encoderDecoderResults?.[1];
+            if (!encoderSource || !decoderSource) {
+                throw new Error('Download interrupted.');
+            }
+        }
+        catch (e) {
+            this.onErrorCallback(e);
+            return;
+        }
+        if (modelName === 'whisperMultilingual') {
+            // The underlying native class is instantiated based on the name of the model. There is no need to
+            // create a separate class for multilingual version of Whisper, since it is the same. We just need
+            // the distinction here, in TS, for start tokens and such. If we introduce
+            // more versions of Whisper, such as the small one, this should be refactored.
+            modelName = AvailableModels.WHISPER;
+        }
+        try {
+            const nativeSpeechToText = await global.loadSpeechToText(encoderSource, decoderSource, modelName);
+            this.speechToTextNativeModule = nativeSpeechToText;
+            this.isReadyCallback(true);
+        }
+        catch (e) {
+            this.onErrorCallback(e);
+        }
+    }
+    configureStreaming(overlapSeconds, windowSize, streamingConfig) {
+        if (streamingConfig) {
+            this.windowSize = MODES[streamingConfig].windowSize * SECOND;
+            this.overlapSeconds = MODES[streamingConfig].overlapSeconds * SECOND;
+        }
+        if (streamingConfig && (windowSize || overlapSeconds)) {
+            Logger.warn(`windowSize and overlapSeconds overrides values from streamingConfig ${streamingConfig}.`);
+        }
+        this.windowSize = (windowSize || 0) * SECOND || this.windowSize;
+        this.overlapSeconds = (overlapSeconds || 0) * SECOND || this.overlapSeconds;
+        if (2 * this.overlapSeconds + this.windowSize >= 30 * SECOND) {
+            Logger.warn(`Invalid values for overlapSeconds and/or windowSize provided. Expected windowSize + 2 * overlapSeconds (== ${this.windowSize + 2 * this.overlapSeconds}) <= 30. Setting windowSize to ${30 * SECOND - 2 * this.overlapSeconds}.`);
+            this.windowSize = 30 * SECOND - 2 * this.overlapSeconds;
+        }
+    }
+    chunkWaveform() {
+        this.numOfChunks = Math.ceil(this.waveform.length / this.windowSize);
+        for (let i = 0; i < this.numOfChunks; i++) {
+            let chunk = [];
+            const left = Math.max(this.windowSize * i - this.overlapSeconds, 0);
+            const right = Math.min(this.windowSize * (i + 1) + this.overlapSeconds, this.waveform.length);
+            chunk = this.waveform.slice(left, right);
+            this.chunks.push(chunk);
+        }
+    }
+    resetState() {
+        this.sequence = [];
+        this.seqs = [];
+        this.waveform = [];
+        this.prevSeq = [];
+        this.chunks = [];
+        this.decodedTranscribeCallback([]);
+        this.onErrorCallback(undefined);
+    }
+    expectedChunkLength() {
+        //only first chunk can be of shorter length, for first chunk there are no seqs decoded
+        return this.seqs.length
+            ? this.windowSize + 2 * this.overlapSeconds
+            : this.windowSize + this.overlapSeconds;
+    }
+    async getStartingTokenIds(audioLanguage) {
+        // We need different starting token ids based on the multilingualism of the model.
+        // The eng version only needs BOS token, while the multilingual one needs:
+        // [BOS, LANG, TRANSCRIBE]. Optionally we should also set notimestamps token, as timestamps
+        // is not yet supported.
+        if (!audioLanguage) {
+            return [this.config.tokenizer.bos];
+        }
+        // FIXME: I should use .getTokenId for the BOS as well, should remove it from config
+        const langTokenId = await this.tokenizerModule.tokenToId(`<|${audioLanguage}|>`);
+        const transcribeTokenId = await this.tokenizerModule.tokenToId('<|transcribe|>');
+        const noTimestampsTokenId = await this.tokenizerModule.tokenToId('<|notimestamps|>');
+        const startingTokenIds = [
+            this.config.tokenizer.bos,
+            langTokenId,
+            transcribeTokenId,
+            noTimestampsTokenId,
+        ];
+        return startingTokenIds;
+    }
+    async decodeChunk(chunk, audioLanguage) {
+        const seq = await this.getStartingTokenIds(audioLanguage);
+        let prevSeqTokenIdx = 0;
+        this.prevSeq = this.sequence.slice();
+        try {
+            await this.encode(new Float32Array(chunk));
+        }
+        catch (error) {
+            this.onErrorCallback(new Error(getError(error) + ' encoding error'));
+            return [];
+        }
+        let lastToken = seq.at(-1);
+        while (lastToken !== this.config.tokenizer.eos) {
+            try {
+                lastToken = await this.decode(seq);
+            }
+            catch (error) {
+                this.onErrorCallback(new Error(getError(error) + ' decoding error'));
+                return [...seq, this.config.tokenizer.eos];
+            }
+            seq.push(lastToken);
+            if (this.seqs.length > 0 &&
+                seq.length < this.seqs.at(-1).length &&
+                seq.length % 3 !== 0) {
+                this.prevSeq.push(this.seqs.at(-1)[prevSeqTokenIdx++]);
+                this.decodedTranscribeCallback(this.prevSeq);
+            }
+        }
+        return seq;
+    }
+    async handleOverlaps(seqs) {
+        const maxInd = longCommonInfPref(seqs.at(-2), seqs.at(-1), HAMMING_DIST_THRESHOLD);
+        this.sequence = [...this.sequence, ...seqs.at(-2).slice(0, maxInd)];
+        this.decodedTranscribeCallback(this.sequence);
+        return this.sequence.slice();
+    }
+    trimLeft(numOfTokensToTrim) {
+        const idx = this.seqs.length - 1;
+        if (this.seqs[idx][0] === this.config.tokenizer.bos) {
+            this.seqs[idx] = this.seqs[idx].slice(numOfTokensToTrim);
+        }
+    }
+    trimRight(numOfTokensToTrim) {
+        const idx = this.seqs.length - 2;
+        if (this.seqs[idx].at(-1) === this.config.tokenizer.eos) {
+            this.seqs[idx] = this.seqs[idx].slice(0, -numOfTokensToTrim);
+        }
+    }
+    // since we are calling this every time (except first) after a new seq is pushed to this.seqs
+    // we can only trim left the last seq and trim right the second to last seq
+    async trimSequences(audioLanguage) {
+        const numSpecialTokens = (await this.getStartingTokenIds(audioLanguage))
+            .length;
+        this.trimLeft(numSpecialTokens + NUM_TOKENS_TO_TRIM);
+        this.trimRight(numSpecialTokens + NUM_TOKENS_TO_TRIM);
+    }
+    // if last chunk is too short combine it with second to last to improve quality
+    validateAndFixLastChunk() {
+        if (this.chunks.length < 2)
+            return;
+        const lastChunkLength = this.chunks.at(-1).length / SECOND;
+        const secondToLastChunkLength = this.chunks.at(-2).length / SECOND;
+        if (lastChunkLength < 5 && secondToLastChunkLength + lastChunkLength < 30) {
+            this.chunks[this.chunks.length - 2] = [
+                ...this.chunks.at(-2).slice(0, -this.overlapSeconds * 2),
+                ...this.chunks.at(-1),
+            ];
+            this.chunks = this.chunks.slice(0, -1);
+        }
+    }
+    async tokenIdsToText(tokenIds) {
+        try {
+            return await this.tokenizerModule.decode(tokenIds, true);
+        }
+        catch (e) {
+            this.onErrorCallback(new Error(`An error has occurred when decoding the token ids: ${e}`));
+            return '';
+        }
+    }
+    async transcribe(waveform, audioLanguage) {
+        try {
+            if (!this.isReady)
+                throw Error(getError(ETError.ModuleNotLoaded));
+            if (this.isGenerating || this.streaming)
+                throw Error(getError(ETError.ModelGenerating));
+            if (!!audioLanguage !== this.config.isMultilingual)
+                throw new Error(getError(ETError.MultilingualConfiguration));
+        }
+        catch (e) {
+            this.onErrorCallback(e);
+            return '';
+        }
+        // Making sure that the error is not set when we get there
+        this.isGeneratingCallback(true);
+        this.resetState();
+        this.waveform = waveform;
+        this.chunkWaveform();
+        this.validateAndFixLastChunk();
+        for (let chunkId = 0; chunkId < this.chunks.length; chunkId++) {
+            const seq = await this.decodeChunk(this.chunks.at(chunkId), audioLanguage);
+            // whole audio is inside one chunk, no processing required
+            if (this.chunks.length === 1) {
+                this.sequence = seq;
+                this.decodedTranscribeCallback(seq);
+                break;
+            }
+            this.seqs.push(seq);
+            if (this.seqs.length < 2)
+                continue;
+            // Remove starting tokenIds and some additional ones
+            await this.trimSequences(audioLanguage);
+            this.prevSeq = await this.handleOverlaps(this.seqs);
+            // last sequence processed
+            // overlaps are already handled, so just append the last seq
+            if (this.seqs.length === this.chunks.length) {
+                this.sequence = [...this.sequence, ...this.seqs.at(-1)];
+                this.decodedTranscribeCallback(this.sequence);
+                this.prevSeq = this.sequence;
+            }
+        }
+        const decodedText = await this.tokenIdsToText(this.sequence);
+        this.isGeneratingCallback(false);
+        return decodedText;
+    }
+    async streamingTranscribe(streamAction, waveform, audioLanguage) {
+        try {
+            if (!this.isReady)
+                throw Error(getError(ETError.ModuleNotLoaded));
+            if (!!audioLanguage !== this.config.isMultilingual)
+                throw new Error(getError(ETError.MultilingualConfiguration));
+            if (streamAction === STREAMING_ACTION.START &&
+                !this.streaming &&
+                this.isGenerating)
+                throw Error(getError(ETError.ModelGenerating));
+            if (streamAction === STREAMING_ACTION.START && this.streaming)
+                throw Error(getError(ETError.ModelGenerating));
+            if (streamAction === STREAMING_ACTION.DATA && !this.streaming)
+                throw Error(getError(ETError.StreamingNotStarted));
+            if (streamAction === STREAMING_ACTION.STOP && !this.streaming)
+                throw Error(getError(ETError.StreamingNotStarted));
+            if (streamAction === STREAMING_ACTION.DATA && !waveform)
+                throw new Error(getError(ETError.MissingDataChunk));
+        }
+        catch (e) {
+            this.onErrorCallback(e);
+            return '';
+        }
+        if (streamAction === STREAMING_ACTION.START) {
+            this.resetState();
+            this.streaming = true;
+            this.isGeneratingCallback(true);
+        }
+        this.waveform = [...this.waveform, ...(waveform || [])];
+        // while buffer has at least required size get chunk and decode
+        while (this.waveform.length >= this.expectedChunkLength()) {
+            const chunk = this.waveform.slice(0, this.windowSize +
+                this.overlapSeconds * (1 + Number(this.seqs.length > 0)));
+            this.chunks = [chunk]; //save last chunk for STREAMING_ACTION.STOP
+            this.waveform = this.waveform.slice(this.windowSize - this.overlapSeconds * Number(this.seqs.length === 0));
+            const seq = await this.decodeChunk(chunk, audioLanguage);
+            this.seqs.push(seq);
+            if (this.seqs.length < 2)
+                continue;
+            await this.trimSequences(audioLanguage);
+            await this.handleOverlaps(this.seqs);
+        }
+        // got final package, process all remaining waveform data
+        // since we run the loop above the waveform has at most one chunk in it
+        if (streamAction === STREAMING_ACTION.STOP) {
+            // pad remaining waveform data with previous chunk to this.windowSize + 2 * this.overlapSeconds
+            const chunk = this.chunks.length
+                ? [
+                    ...this.chunks[0].slice(0, this.windowSize),
+                    ...this.waveform,
+                ].slice(-this.windowSize - 2 * this.overlapSeconds)
+                : this.waveform;
+            this.waveform = [];
+            const seq = await this.decodeChunk(chunk, audioLanguage);
+            this.seqs.push(seq);
+            if (this.seqs.length === 1) {
+                this.sequence = this.seqs[0];
+            }
+            else {
+                await this.trimSequences(audioLanguage);
+                await this.handleOverlaps(this.seqs);
+                this.sequence = [...this.sequence, ...this.seqs.at(-1)];
+            }
+            this.decodedTranscribeCallback(this.sequence);
+            this.isGeneratingCallback(false);
+            this.streaming = false;
+        }
+        const decodedText = await this.tokenIdsToText(this.sequence);
+        return decodedText;
+    }
+    async encode(waveform) {
+        return await this.speechToTextNativeModule.encode(waveform);
+    }
+    async decode(seq) {
+        return await this.speechToTextNativeModule.decode(seq);
+    }
+}

package/lib/controllers/VerticalOCRController.d.ts

@@ -0,0 +1,25 @@
+import { ResourceSource } from '../types/common';
+import { OCRLanguage } from '../types/ocr';
+export declare class VerticalOCRController {
+    private ocrNativeModule;
+    isReady: boolean;
+    isGenerating: boolean;
+    error: string | null;
+    private isReadyCallback;
+    private isGeneratingCallback;
+    private errorCallback;
+    constructor({ isReadyCallback, isGeneratingCallback, errorCallback, }?: {
+        isReadyCallback?: ((_isReady: boolean) => void) | undefined;
+        isGeneratingCallback?: ((_isGenerating: boolean) => void) | undefined;
+        errorCallback?: ((_error: string) => void) | undefined;
+    });
+    load: (detectorSources: {
+        detectorLarge: ResourceSource;
+        detectorNarrow: ResourceSource;
+    }, recognizerSources: {
+        recognizerLarge: ResourceSource;
+        recognizerSmall: ResourceSource;
+    }, language: OCRLanguage, independentCharacters: boolean, onDownloadProgressCallback: (downloadProgress: number) => void) => Promise<void>;
+    forward: (input: string) => Promise<any>;
+    delete(): void;
+}

package/lib/controllers/VerticalOCRController.js

@@ -0,0 +1,75 @@
+import { symbols } from '../constants/ocr/symbols';
+import { ETError, getError } from '../Error';
+import { ResourceFetcher } from '../utils/ResourceFetcher';
+export class VerticalOCRController {
+    ocrNativeModule;
+    isReady = false;
+    isGenerating = false;
+    error = null;
+    isReadyCallback;
+    isGeneratingCallback;
+    errorCallback;
+    constructor({ isReadyCallback = (_isReady) => { }, isGeneratingCallback = (_isGenerating) => { }, errorCallback = (_error) => { }, } = {}) {
+        this.isReadyCallback = isReadyCallback;
+        this.isGeneratingCallback = isGeneratingCallback;
+        this.errorCallback = errorCallback;
+    }
+    load = async (detectorSources, recognizerSources, language, independentCharacters, onDownloadProgressCallback) => {
+        try {
+            if (Object.keys(detectorSources).length !== 2 ||
+                Object.keys(recognizerSources).length !== 2)
+                return;
+            if (!symbols[language]) {
+                throw new Error(getError(ETError.LanguageNotSupported));
+            }
+            this.isReady = false;
+            this.isReadyCallback(this.isReady);
+            const paths = await ResourceFetcher.fetch(onDownloadProgressCallback, detectorSources.detectorLarge, detectorSources.detectorNarrow, independentCharacters
+                ? recognizerSources.recognizerSmall
+                : recognizerSources.recognizerLarge);
+            if (paths === null || paths.length < 3) {
+                throw new Error('Download interrupted');
+            }
+            this.ocrNativeModule = global.loadVerticalOCR(paths[0], paths[1], paths[2], symbols[language], independentCharacters);
+            this.isReady = true;
+            this.isReadyCallback(this.isReady);
+        }
+        catch (e) {
+            if (this.errorCallback) {
+                this.errorCallback(getError(e));
+            }
+            else {
+                throw new Error(getError(e));
+            }
+        }
+    };
+    forward = async (input) => {
+        if (!this.isReady) {
+            throw new Error(getError(ETError.ModuleNotLoaded));
+        }
+        if (this.isGenerating) {
+            throw new Error(getError(ETError.ModelGenerating));
+        }
+        try {
+            this.isGenerating = true;
+            this.isGeneratingCallback(this.isGenerating);
+            return await this.ocrNativeModule.generate(input);
+        }
+        catch (e) {
+            throw new Error(getError(e));
+        }
+        finally {
+            this.isGenerating = false;
+            this.isGeneratingCallback(this.isGenerating);
+        }
+    };
+    delete() {
+        if (this.isGenerating) {
+            throw new Error(getError(ETError.ModelGenerating) +
+                'You cannot delete the model. You must wait until the generating is finished.');
+        }
+        this.ocrNativeModule.unload();
+        this.isReadyCallback(false);
+        this.isGeneratingCallback(false);
+    }
+}

package/lib/hooks/computer_vision/useClassification.d.ts

@@ -0,0 +1,15 @@
+import { ResourceSource } from '../../types/common';
+interface Props {
+    model: {
+        modelSource: ResourceSource;
+    };
+    preventLoad?: boolean;
+}
+export declare const useClassification: ({ model, preventLoad }: Props) => {
+    error: string | null;
+    isReady: boolean;
+    isGenerating: boolean;
+    downloadProgress: number;
+    forward: (imageSource: string) => Promise<any>;
+};
+export {};

package/lib/hooks/computer_vision/useClassification.js

@@ -0,0 +1,7 @@
+import { useNonStaticModule } from '../useNonStaticModule';
+import { ClassificationModule } from '../../modules/computer_vision/ClassificationModule';
+export const useClassification = ({ model, preventLoad = false }) => useNonStaticModule({
+    module: ClassificationModule,
+    model,
+    preventLoad: preventLoad,
+});

package/lib/hooks/computer_vision/useImageEmbeddings.d.ts

@@ -0,0 +1,15 @@
+import { ResourceSource } from '../../types/common';
+interface Props {
+    model: {
+        modelSource: ResourceSource;
+    };
+    preventLoad?: boolean;
+}
+export declare const useImageEmbeddings: ({ model, preventLoad }: Props) => {
+    error: string | null;
+    isReady: boolean;
+    isGenerating: boolean;
+    downloadProgress: number;
+    forward: (imageSource: string) => Promise<Float32Array<ArrayBufferLike>>;
+};
+export {};

package/lib/hooks/computer_vision/useImageEmbeddings.js

@@ -0,0 +1,7 @@
+import { ImageEmbeddingsModule } from '../../modules/computer_vision/ImageEmbeddingsModule';
+import { useNonStaticModule } from '../useNonStaticModule';
+export const useImageEmbeddings = ({ model, preventLoad = false }) => useNonStaticModule({
+    module: ImageEmbeddingsModule,
+    model,
+    preventLoad,
+});

package/lib/hooks/computer_vision/useImageSegmentation.d.ts

@@ -0,0 +1,38 @@
+import { ResourceSource } from '../../types/common';
+interface Props {
+    model: {
+        modelSource: ResourceSource;
+    };
+    preventLoad?: boolean;
+}
+export declare const useImageSegmentation: ({ model, preventLoad }: Props) => {
+    error: string | null;
+    isReady: boolean;
+    isGenerating: boolean;
+    downloadProgress: number;
+    forward: (imageSource: string, classesOfInterest?: import("../..").DeeplabLabel[] | undefined, resize?: boolean | undefined) => Promise<{
+        0?: number[] | undefined;
+        1?: number[] | undefined;
+        2?: number[] | undefined;
+        3?: number[] | undefined;
+        4?: number[] | undefined;
+        5?: number[] | undefined;
+        6?: number[] | undefined;
+        7?: number[] | undefined;
+        8?: number[] | undefined;
+        9?: number[] | undefined;
+        10?: number[] | undefined;
+        11?: number[] | undefined;
+        12?: number[] | undefined;
+        13?: number[] | undefined;
+        14?: number[] | undefined;
+        15?: number[] | undefined;
+        16?: number[] | undefined;
+        17?: number[] | undefined;
+        18?: number[] | undefined;
+        19?: number[] | undefined;
+        20?: number[] | undefined;
+        21?: number[] | undefined;
+    }>;
+};
+export {};

package/lib/hooks/computer_vision/useImageSegmentation.js

@@ -0,0 +1,7 @@
+import { useNonStaticModule } from '../useNonStaticModule';
+import { ImageSegmentationModule } from '../../modules/computer_vision/ImageSegmentationModule';
+export const useImageSegmentation = ({ model, preventLoad = false }) => useNonStaticModule({
+    module: ImageSegmentationModule,
+    model,
+    preventLoad,
+});

package/lib/hooks/computer_vision/useOCR.d.ts

@@ -0,0 +1,20 @@
+import { ResourceSource } from '../../types/common';
+import { OCRDetection, OCRLanguage } from '../../types/ocr';
+interface OCRModule {
+    error: string | null;
+    isReady: boolean;
+    isGenerating: boolean;
+    forward: (input: string) => Promise<OCRDetection[]>;
+    downloadProgress: number;
+}
+export declare const useOCR: ({ model, preventLoad, }: {
+    model: {
+        detectorSource: ResourceSource;
+        recognizerLarge: ResourceSource;
+        recognizerMedium: ResourceSource;
+        recognizerSmall: ResourceSource;
+        language: OCRLanguage;
+    };
+    preventLoad?: boolean;
+}) => OCRModule;
+export {};

package/lib/hooks/computer_vision/useOCR.js

@@ -0,0 +1,42 @@
+import { useEffect, useMemo, useState } from 'react';
+import { OCRController } from '../../controllers/OCRController';
+export const useOCR = ({ model, preventLoad = false, }) => {
+    const [error, setError] = useState(null);
+    const [isReady, setIsReady] = useState(false);
+    const [isGenerating, setIsGenerating] = useState(false);
+    const [downloadProgress, setDownloadProgress] = useState(0);
+    const controllerInstance = useMemo(() => new OCRController({
+        isReadyCallback: setIsReady,
+        isGeneratingCallback: setIsGenerating,
+        errorCallback: setError,
+    }), []);
+    useEffect(() => {
+        if (preventLoad)
+            return;
+        (async () => {
+            await controllerInstance.load(model.detectorSource, {
+                recognizerLarge: model.recognizerLarge,
+                recognizerMedium: model.recognizerMedium,
+                recognizerSmall: model.recognizerSmall,
+            }, model.language, setDownloadProgress);
+        })();
+        return () => {
+            controllerInstance.delete();
+        };
+    }, [
+        controllerInstance,
+        model.detectorSource,
+        model.recognizerLarge,
+        model.recognizerMedium,
+        model.recognizerSmall,
+        model.language,
+        preventLoad,
+    ]);
+    return {
+        error,
+        isReady,
+        isGenerating,
+        forward: controllerInstance.forward,
+        downloadProgress,
+    };
+};

package/lib/hooks/computer_vision/useObjectDetection.d.ts

@@ -0,0 +1,15 @@
+import { ResourceSource } from '../../types/common';
+interface Props {
+    model: {
+        modelSource: ResourceSource;
+    };
+    preventLoad?: boolean;
+}
+export declare const useObjectDetection: ({ model, preventLoad }: Props) => {
+    error: string | null;
+    isReady: boolean;
+    isGenerating: boolean;
+    downloadProgress: number;
+    forward: (imageSource: string, detectionThreshold?: number | undefined) => Promise<import("../..").Detection[]>;
+};
+export {};

package/lib/hooks/computer_vision/useObjectDetection.js

@@ -0,0 +1,7 @@
+import { useNonStaticModule } from '../useNonStaticModule';
+import { ObjectDetectionModule } from '../../modules/computer_vision/ObjectDetectionModule';
+export const useObjectDetection = ({ model, preventLoad = false }) => useNonStaticModule({
+    module: ObjectDetectionModule,
+    model,
+    preventLoad: preventLoad,
+});

package/lib/hooks/computer_vision/useStyleTransfer.d.ts

@@ -0,0 +1,15 @@
+import { ResourceSource } from '../../types/common';
+interface Props {
+    model: {
+        modelSource: ResourceSource;
+    };
+    preventLoad?: boolean;
+}
+export declare const useStyleTransfer: ({ model, preventLoad }: Props) => {
+    error: string | null;
+    isReady: boolean;
+    isGenerating: boolean;
+    downloadProgress: number;
+    forward: (imageSource: string) => Promise<string>;
+};
+export {};

package/lib/hooks/computer_vision/useStyleTransfer.js

@@ -0,0 +1,7 @@
+import { useNonStaticModule } from '../useNonStaticModule';
+import { StyleTransferModule } from '../../modules/computer_vision/StyleTransferModule';
+export const useStyleTransfer = ({ model, preventLoad = false }) => useNonStaticModule({
+    module: StyleTransferModule,
+    model,
+    preventLoad: preventLoad,
+});