react-native-executorch 0.4.9 → 0.5.0-nightly-6f4dd53-20251211

package/src/controllers/VerticalOCRController.ts

@@ -1,34 +1,29 @@
 import { symbols } from '../constants/ocr/symbols';
 import { ETError, getError } from '../Error';
-import { VerticalOCRNativeModule } from '../native/RnExecutorchModules';
 import { ResourceSource } from '../types/common';
 import { OCRLanguage } from '../types/ocr';
 import { ResourceFetcher } from '../utils/ResourceFetcher';
 
 export class VerticalOCRController {
-  private ocrNativeModule: typeof VerticalOCRNativeModule;
+  private ocrNativeModule: any;
   public isReady: boolean = false;
   public isGenerating: boolean = false;
   public error: string | null = null;
-  private modelDownloadProgressCallback: (downloadProgress: number) => void;
   private isReadyCallback: (isReady: boolean) => void;
   private isGeneratingCallback: (isGenerating: boolean) => void;
   private errorCallback: (error: string) => void;
 
   constructor({
-    modelDownloadProgressCallback = (_downloadProgress: number) => {},
     isReadyCallback = (_isReady: boolean) => {},
     isGeneratingCallback = (_isGenerating: boolean) => {},
     errorCallback = (_error: string) => {},
-  }) {
-    this.ocrNativeModule = VerticalOCRNativeModule;
-    this.modelDownloadProgressCallback = modelDownloadProgressCallback;
+  } = {}) {
     this.isReadyCallback = isReadyCallback;
     this.isGeneratingCallback = isGeneratingCallback;
     this.errorCallback = errorCallback;
   }
 
-  public loadModel = async (
+  public load = async (
     detectorSources: {
       detectorLarge: ResourceSource;
       detectorNarrow: ResourceSource;
@@ -38,7 +33,8 @@ export class VerticalOCRController {
       recognizerSmall: ResourceSource;
     },
     language: OCRLanguage,
-    independentCharacters: boolean
+    independentCharacters: boolean,
+    onDownloadProgressCallback: (downloadProgress: number) => void
   ) => {
     try {
       if (
@@ -54,16 +50,18 @@ export class VerticalOCRController {
       this.isReady = false;
       this.isReadyCallback(this.isReady);
 
-      const paths = await ResourceFetcher.fetchMultipleResources(
-        this.modelDownloadProgressCallback,
+      const paths = await ResourceFetcher.fetch(
+        onDownloadProgressCallback,
         detectorSources.detectorLarge,
         detectorSources.detectorNarrow,
         independentCharacters
           ? recognizerSources.recognizerSmall
           : recognizerSources.recognizerLarge
       );
-
-      await this.ocrNativeModule.loadModule(
+      if (paths === null || paths.length < 3) {
+        throw new Error('Download interrupted');
+      }
+      this.ocrNativeModule = global.loadVerticalOCR(
         paths[0]!,
         paths[1]!,
         paths[2]!,
@@ -82,7 +80,7 @@ export class VerticalOCRController {
     }
   };
 
-  public forward = async (input: string) => {
+  public forward = async (imageSource: string) => {
     if (!this.isReady) {
       throw new Error(getError(ETError.ModuleNotLoaded));
     }
@@ -93,7 +91,7 @@ export class VerticalOCRController {
     try {
       this.isGenerating = true;
       this.isGeneratingCallback(this.isGenerating);
-      return await this.ocrNativeModule.forward(input);
+      return await this.ocrNativeModule.generate(imageSource);
     } catch (e) {
       throw new Error(getError(e));
     } finally {
@@ -101,4 +99,16 @@ export class VerticalOCRController {
       this.isGeneratingCallback(this.isGenerating);
     }
   };
+
+  public delete() {
+    if (this.isGenerating) {
+      throw new Error(
+        getError(ETError.ModelGenerating) +
+          'You cannot delete the model. You must wait until the generating is finished.'
+      );
+    }
+    this.ocrNativeModule.unload();
+    this.isReadyCallback(false);
+    this.isGeneratingCallback(false);
+  }
 }

package/src/hooks/computer_vision/useClassification.ts

@@ -1,16 +1,15 @@
-import { ClassificationModule } from '../../modules/computer_vision/ClassificationModule';
 import { ResourceSource } from '../../types/common';
 import { useModule } from '../useModule';
+import { ClassificationModule } from '../../modules/computer_vision/ClassificationModule';
 
-export const useClassification = ({
-  modelSource,
-  preventLoad = false,
-}: {
-  modelSource: ResourceSource;
+interface Props {
+  model: { modelSource: ResourceSource };
   preventLoad?: boolean;
-}) =>
+}
+
+export const useClassification = ({ model, preventLoad = false }: Props) =>
   useModule({
     module: ClassificationModule,
-    loadArgs: [modelSource],
-    preventLoad,
+    model,
+    preventLoad: preventLoad,
   });

package/src/hooks/computer_vision/useImageEmbeddings.ts

@@ -0,0 +1,15 @@
+import { ImageEmbeddingsModule } from '../../modules/computer_vision/ImageEmbeddingsModule';
+import { ResourceSource } from '../../types/common';
+import { useModule } from '../useModule';
+
+interface Props {
+  model: { modelSource: ResourceSource };
+  preventLoad?: boolean;
+}
+
+export const useImageEmbeddings = ({ model, preventLoad = false }: Props) =>
+  useModule({
+    module: ImageEmbeddingsModule,
+    model,
+    preventLoad,
+  });

package/src/hooks/computer_vision/useImageSegmentation.ts

@@ -3,16 +3,13 @@ import { ImageSegmentationModule } from '../../modules/computer_vision/ImageSegm
 import { ResourceSource } from '../../types/common';
 
 interface Props {
-  modelSource: ResourceSource;
+  model: { modelSource: ResourceSource };
   preventLoad?: boolean;
 }
 
-export const useImageSegmentation = ({
-  modelSource,
-  preventLoad = false,
-}: Props) =>
+export const useImageSegmentation = ({ model, preventLoad = false }: Props) =>
   useModule({
     module: ImageSegmentationModule,
-    loadArgs: [modelSource],
+    model,
     preventLoad,
   });

package/src/hooks/computer_vision/useOCR.ts

@@ -1,4 +1,4 @@
-import { useEffect, useMemo, useState } from 'react';
+import { useEffect, useState } from 'react';
 import { ResourceSource } from '../../types/common';
 import { OCRDetection, OCRLanguage } from '../../types/ocr';
 import { OCRController } from '../../controllers/OCRController';
@@ -7,23 +7,21 @@ interface OCRModule {
   error: string | null;
   isReady: boolean;
   isGenerating: boolean;
-  forward: (input: string) => Promise<OCRDetection[]>;
+  forward: (imageSource: string) => Promise<OCRDetection[]>;
   downloadProgress: number;
 }
 
 export const useOCR = ({
-  detectorSource,
-  recognizerSources,
-  language = 'en',
+  model,
   preventLoad = false,
 }: {
-  detectorSource: ResourceSource;
-  recognizerSources: {
+  model: {
+    detectorSource: ResourceSource;
     recognizerLarge: ResourceSource;
     recognizerMedium: ResourceSource;
     recognizerSmall: ResourceSource;
+    language: OCRLanguage;
   };
-  language?: OCRLanguage;
   preventLoad?: boolean;
 }): OCRModule => {
   const [error, setError] = useState<string | null>(null);
@@ -31,32 +29,41 @@ export const useOCR = ({
   const [isGenerating, setIsGenerating] = useState(false);
   const [downloadProgress, setDownloadProgress] = useState(0);
 
-  const model = useMemo(
+  const [controllerInstance] = useState(
     () =>
       new OCRController({
-        modelDownloadProgressCallback: setDownloadProgress,
         isReadyCallback: setIsReady,
         isGeneratingCallback: setIsGenerating,
         errorCallback: setError,
-      }),
-    []
+      })
   );
 
   useEffect(() => {
-    const loadModel = async () => {
-      await model.loadModel(detectorSource, recognizerSources, language);
-    };
+    if (preventLoad) return;
+
+    (async () => {
+      await controllerInstance.load(
+        model.detectorSource,
+        {
+          recognizerLarge: model.recognizerLarge,
+          recognizerMedium: model.recognizerMedium,
+          recognizerSmall: model.recognizerSmall,
+        },
+        model.language,
+        setDownloadProgress
+      );
+    })();
 
-    if (!preventLoad) {
-      loadModel();
-    }
-    // eslint-disable-next-line react-hooks/exhaustive-deps
+    return () => {
+      controllerInstance.delete();
+    };
   }, [
-    model,
-    detectorSource,
-    language,
-    // eslint-disable-next-line react-hooks/exhaustive-deps
-    JSON.stringify(recognizerSources),
+    controllerInstance,
+    model.detectorSource,
+    model.recognizerLarge,
+    model.recognizerMedium,
+    model.recognizerSmall,
+    model.language,
     preventLoad,
   ]);
 
@@ -64,7 +71,7 @@ export const useOCR = ({
     error,
     isReady,
     isGenerating,
-    forward: model.forward,
+    forward: controllerInstance.forward,
     downloadProgress,
   };
 };

package/src/hooks/computer_vision/useObjectDetection.ts

@@ -3,16 +3,13 @@ import { useModule } from '../useModule';
 import { ObjectDetectionModule } from '../../modules/computer_vision/ObjectDetectionModule';
 
 interface Props {
-  modelSource: ResourceSource;
+  model: { modelSource: ResourceSource };
   preventLoad?: boolean;
 }
 
-export const useObjectDetection = ({
-  modelSource,
-  preventLoad = false,
-}: Props) =>
+export const useObjectDetection = ({ model, preventLoad = false }: Props) =>
   useModule({
     module: ObjectDetectionModule,
-    loadArgs: [modelSource],
-    preventLoad,
+    model,
+    preventLoad: preventLoad,
   });

package/src/hooks/computer_vision/useStyleTransfer.ts

@@ -3,13 +3,13 @@ import { useModule } from '../useModule';
 import { StyleTransferModule } from '../../modules/computer_vision/StyleTransferModule';
 
 interface Props {
-  modelSource: ResourceSource;
+  model: { modelSource: ResourceSource };
   preventLoad?: boolean;
 }
 
-export const useStyleTransfer = ({ modelSource, preventLoad = false }: Props) =>
+export const useStyleTransfer = ({ model, preventLoad = false }: Props) =>
   useModule({
     module: StyleTransferModule,
-    loadArgs: [modelSource],
-    preventLoad,
+    model,
+    preventLoad: preventLoad,
   });

package/src/hooks/computer_vision/useTextToImage.ts

@@ -0,0 +1,92 @@
+import { useCallback, useEffect, useState } from 'react';
+import { ETError, getError } from '../../Error';
+import { ResourceSource } from '../../types/common';
+import { TextToImageModule } from '../../modules/computer_vision/TextToImageModule';
+
+interface TextToImageType {
+  isReady: boolean;
+  isGenerating: boolean;
+  downloadProgress: number;
+  error: string | null;
+  generate: (
+    input: string,
+    imageSize?: number,
+    numSteps?: number,
+    seed?: number
+  ) => Promise<string>;
+  interrupt: () => void;
+}
+
+export const useTextToImage = ({
+  model,
+  inferenceCallback,
+  preventLoad = false,
+}: {
+  model: {
+    tokenizerSource: ResourceSource;
+    schedulerSource: ResourceSource;
+    encoderSource: ResourceSource;
+    unetSource: ResourceSource;
+    decoderSource: ResourceSource;
+  };
+  inferenceCallback?: (stepIdx: number) => void;
+  preventLoad?: boolean;
+}): TextToImageType => {
+  const [isReady, setIsReady] = useState(false);
+  const [isGenerating, setIsGenerating] = useState(false);
+  const [downloadProgress, setDownloadProgress] = useState(0);
+  const [error, setError] = useState<string | null>(null);
+
+  const [module] = useState(() => new TextToImageModule(inferenceCallback));
+
+  useEffect(() => {
+    if (preventLoad) return;
+
+    (async () => {
+      setDownloadProgress(0);
+      setError(null);
+      try {
+        setIsReady(false);
+        await module.load(model, setDownloadProgress);
+        setIsReady(true);
+      } catch (err) {
+        setError((err as Error).message);
+      }
+    })();
+
+    return () => {
+      module.delete();
+    };
+  }, [module, model, preventLoad]);
+
+  const generate = async (
+    input: string,
+    imageSize?: number,
+    numSteps?: number,
+    seed?: number
+  ): Promise<string> => {
+    if (!isReady) throw new Error(getError(ETError.ModuleNotLoaded));
+    if (isGenerating) throw new Error(getError(ETError.ModelGenerating));
+    try {
+      setIsGenerating(true);
+      return await module.forward(input, imageSize, numSteps, seed);
+    } finally {
+      setIsGenerating(false);
+    }
+  };
+
+  const interrupt = useCallback(() => {
+    if (isGenerating) {
+      module.interrupt();
+    }
+  }, [module, isGenerating]);
+
+  return {
+    isReady,
+    isGenerating,
+    downloadProgress,
+    error,
+    generate,
+    interrupt,
+  };
+};

package/src/hooks/computer_vision/useVerticalOCR.ts

@@ -1,4 +1,4 @@
-import { useEffect, useMemo, useState } from 'react';
+import { useEffect, useState } from 'react';
 import { ResourceSource } from '../../types/common';
 import { OCRDetection, OCRLanguage } from '../../types/ocr';
 import { VerticalOCRController } from '../../controllers/VerticalOCRController';
@@ -7,26 +7,22 @@ interface OCRModule {
   error: string | null;
   isReady: boolean;
   isGenerating: boolean;
-  forward: (input: string) => Promise<OCRDetection[]>;
+  forward: (imageSource: string) => Promise<OCRDetection[]>;
   downloadProgress: number;
 }
 
 export const useVerticalOCR = ({
-  detectorSources,
-  recognizerSources,
-  language = 'en',
+  model,
   independentCharacters = false,
   preventLoad = false,
 }: {
-  detectorSources: {
+  model: {
     detectorLarge: ResourceSource;
     detectorNarrow: ResourceSource;
-  };
-  recognizerSources: {
     recognizerLarge: ResourceSource;
     recognizerSmall: ResourceSource;
+    language: OCRLanguage;
   };
-  language?: OCRLanguage;
   independentCharacters?: boolean;
   preventLoad?: boolean;
 }): OCRModule => {
@@ -35,39 +31,45 @@ export const useVerticalOCR = ({
   const [isGenerating, setIsGenerating] = useState(false);
   const [downloadProgress, setDownloadProgress] = useState(0);
 
-  const model = useMemo(
+  const [controllerInstance] = useState(
     () =>
       new VerticalOCRController({
-        modelDownloadProgressCallback: setDownloadProgress,
         isReadyCallback: setIsReady,
         isGeneratingCallback: setIsGenerating,
         errorCallback: setError,
-      }),
-    []
+      })
   );
 
   useEffect(() => {
-    const loadModel = async () => {
-      await model.loadModel(
-        detectorSources,
-        recognizerSources,
-        language,
-        independentCharacters
+    if (preventLoad) return;
+
+    (async () => {
+      await controllerInstance.load(
+        {
+          detectorLarge: model.detectorLarge,
+          detectorNarrow: model.detectorNarrow,
+        },
+        {
+          recognizerLarge: model.recognizerLarge,
+          recognizerSmall: model.recognizerSmall,
+        },
+        model.language,
+        independentCharacters,
+        setDownloadProgress
       );
-    };
+    })();
 
-    if (!preventLoad) {
-      loadModel();
-    }
-    // eslint-disable-next-line react-hooks/exhaustive-deps
+    return () => {
+      controllerInstance.delete();
+    };
   }, [
-    model,
-    // eslint-disable-next-line react-hooks/exhaustive-deps
-    JSON.stringify(detectorSources),
-    language,
+    controllerInstance,
+    model.detectorLarge,
+    model.detectorNarrow,
+    model.recognizerLarge,
+    model.recognizerSmall,
+    model.language,
     independentCharacters,
-    // eslint-disable-next-line react-hooks/exhaustive-deps
-    JSON.stringify(recognizerSources),
     preventLoad,
   ]);
 
@@ -75,7 +77,7 @@ export const useVerticalOCR = ({
     error,
     isReady,
     isGenerating,
-    forward: model.forward,
+    forward: controllerInstance.forward,
     downloadProgress,
   };
 };

package/src/hooks/general/useExecutorchModule.ts

@@ -13,6 +13,6 @@ export const useExecutorchModule = ({
 }: Props) =>
   useModule({
     module: ExecutorchModule,
-    loadArgs: [modelSource],
+    model: modelSource,
     preventLoad,
   });

package/src/hooks/natural_language_processing/useLLM.ts

@@ -1,7 +1,8 @@
-import { useCallback, useEffect, useMemo, useState } from 'react';
+import { useCallback, useEffect, useState } from 'react';
 import { ResourceSource } from '../../types/common';
 import {
   ChatConfig,
+  GenerationConfig,
   LLMTool,
   LLMType,
   Message,
@@ -13,14 +14,14 @@ import { LLMController } from '../../controllers/LLMController';
 Hook version of LLMModule
 */
 export const useLLM = ({
-  modelSource,
-  tokenizerSource,
-  tokenizerConfigSource,
+  model,
   preventLoad = false,
 }: {
-  modelSource: ResourceSource;
-  tokenizerSource: ResourceSource;
-  tokenizerConfigSource: ResourceSource;
+  model: {
+    modelSource: ResourceSource;
+    tokenizerSource: ResourceSource;
+    tokenizerConfigSource: ResourceSource;
+  };
   preventLoad?: boolean;
 }): LLMType => {
   const [token, setToken] = useState<string>('');
@@ -36,74 +37,95 @@ export const useLLM = ({
     setResponse((prevResponse) => prevResponse + newToken);
   }, []);
 
-  const model = useMemo(
+  const [controllerInstance] = useState(
     () =>
       new LLMController({
         tokenCallback: tokenCallback,
         messageHistoryCallback: setMessageHistory,
         isReadyCallback: setIsReady,
         isGeneratingCallback: setIsGenerating,
-        onDownloadProgressCallback: setDownloadProgress,
-      }),
-    [tokenCallback]
+      })
   );
 
   useEffect(() => {
     setDownloadProgress(0);
     setError(null);
 
-    if (!preventLoad) {
+    if (preventLoad) return;
+
+    (async () => {
       try {
-        (async () => {
-          await model.load({
-            modelSource,
-            tokenizerSource,
-            tokenizerConfigSource,
-          });
-        })();
+        await controllerInstance.load({
+          modelSource: model.modelSource,
+          tokenizerSource: model.tokenizerSource,
+          tokenizerConfigSource: model.tokenizerConfigSource,
+          onDownloadProgressCallback: setDownloadProgress,
+        });
       } catch (e) {
         setError(e);
       }
-    }
+    })();
 
     return () => {
-      model.delete();
+      controllerInstance.delete();
     };
-  }, [modelSource, tokenizerSource, tokenizerConfigSource, preventLoad, model]);
+  }, [
+    controllerInstance,
+    model.modelSource,
+    model.tokenizerSource,
+    model.tokenizerConfigSource,
+    preventLoad,
+  ]);
 
   // memoization of returned functions
   const configure = useCallback(
     ({
       chatConfig,
       toolsConfig,
+      generationConfig,
     }: {
       chatConfig?: Partial<ChatConfig>;
       toolsConfig?: ToolsConfig;
-    }) => model.configure({ chatConfig, toolsConfig }),
-    [model]
+      generationConfig?: GenerationConfig;
+    }) =>
+      controllerInstance.configure({
+        chatConfig,
+        toolsConfig,
+        generationConfig,
+      }),
+    [controllerInstance]
   );
 
   const generate = useCallback(
     (messages: Message[], tools?: LLMTool[]) => {
       setResponse('');
-      return model.generate(messages, tools);
+      return controllerInstance.generate(messages, tools);
     },
-    [model]
+    [controllerInstance]
   );
 
   const sendMessage = useCallback(
     (message: string) => {
       setResponse('');
-      return model.sendMessage(message);
+      return controllerInstance.sendMessage(message);
     },
-    [model]
+    [controllerInstance]
   );
 
   const deleteMessage = useCallback(
-    (index: number) => model.deleteMessage(index),
-    [model]
+    (index: number) => controllerInstance.deleteMessage(index),
+    [controllerInstance]
+  );
+
+  const interrupt = useCallback(
+    () => controllerInstance.interrupt(),
+    [controllerInstance]
+  );
+
+  const getGeneratedTokenCount = useCallback(
+    () => controllerInstance.getGeneratedTokenCount(),
+    [controllerInstance]
   );
-  const interrupt = useCallback(() => model.interrupt(), [model]);
 
   return {
     messageHistory,
@@ -113,6 +135,7 @@ export const useLLM = ({
     isGenerating,
     downloadProgress,
     error,
+    getGeneratedTokenCount: getGeneratedTokenCount,
     configure: configure,
     generate: generate,
     sendMessage: sendMessage,