react-native-executorch 0.4.10 → 0.5.0-nightly-6f4dd53-20251211

package/src/modules/computer_vision/ImageSegmentationModule.ts

@@ -1,39 +1,47 @@
-import { BaseModule } from '../BaseModule';
-import { getError } from '../../Error';
-import { DeeplabLabel } from '../../types/imageSegmentation';
+import { ResourceFetcher } from '../../utils/ResourceFetcher';
 import { ResourceSource } from '../../types/common';
-import { ImageSegmentationNativeModule } from '../../native/RnExecutorchModules';
+import { DeeplabLabel } from '../../types/imageSegmentation';
+import { ETError, getError } from '../../Error';
+import { BaseModule } from '../BaseModule';
 
 export class ImageSegmentationModule extends BaseModule {
-  protected static override nativeModule = ImageSegmentationNativeModule;
-
-  static override async load(modelSource: ResourceSource) {
-    return await super.load(modelSource);
+  async load(
+    model: { modelSource: ResourceSource },
+    onDownloadProgressCallback: (progress: number) => void = () => {}
+  ): Promise<void> {
+    const paths = await ResourceFetcher.fetch(
+      onDownloadProgressCallback,
+      model.modelSource
+    );
+    if (paths === null || paths.length < 1) {
+      throw new Error('Download interrupted.');
+    }
+    this.nativeModule = global.loadImageSegmentation(paths[0] || '');
   }
 
-  static override async forward(
-    input: string,
+  async forward(
+    imageSource: string,
     classesOfInterest?: DeeplabLabel[],
     resize?: boolean
-  ) {
-    try {
-      const stringDict = await (this.nativeModule.forward(
-        input,
-        (classesOfInterest || []).map((label) => DeeplabLabel[label]),
-        resize || false
-      ) as ReturnType<(typeof this.nativeModule)['forward']>);
+  ): Promise<{ [key in DeeplabLabel]?: number[] }> {
+    if (this.nativeModule == null) {
+      throw new Error(getError(ETError.ModuleNotLoaded));
+    }
+
+    const stringDict = await this.nativeModule.generate(
+      imageSource,
+      (classesOfInterest || []).map((label) => DeeplabLabel[label]),
+      resize || false
+    );
 
-      let enumDict: { [key in DeeplabLabel]?: number[] } = {};
+    let enumDict: { [key in DeeplabLabel]?: number[] } = {};
 
-      for (const key in stringDict) {
-        if (key in DeeplabLabel) {
-          const enumKey = DeeplabLabel[key as keyof typeof DeeplabLabel];
-          enumDict[enumKey] = stringDict[key];
-        }
+    for (const key in stringDict) {
+      if (key in DeeplabLabel) {
+        const enumKey = DeeplabLabel[key as keyof typeof DeeplabLabel];
+        enumDict[enumKey] = stringDict[key];
       }
-      return enumDict;
-    } catch (e) {
-      throw new Error(getError(e));
     }
+    return enumDict;
   }
 }

package/src/modules/computer_vision/OCRModule.ts

@@ -3,31 +3,39 @@ import { ResourceSource } from '../../types/common';
 import { OCRLanguage } from '../../types/ocr';
 
 export class OCRModule {
-  static module: OCRController;
+  private controller: OCRController;
 
-  static onDownloadProgressCallback = (_downloadProgress: number) => {};
+  constructor() {
+    this.controller = new OCRController();
+  }
 
-  static async load(
-    detectorSource: ResourceSource,
-    recognizerSources: {
+  async load(
+    model: {
+      detectorSource: ResourceSource;
       recognizerLarge: ResourceSource;
       recognizerMedium: ResourceSource;
       recognizerSmall: ResourceSource;
+      language: OCRLanguage;
     },
-    language: OCRLanguage = 'en'
+    onDownloadProgressCallback: (progress: number) => void = () => {}
   ) {
-    this.module = new OCRController({
-      modelDownloadProgressCallback: this.onDownloadProgressCallback,
-    });
-
-    await this.module.loadModel(detectorSource, recognizerSources, language);
+    await this.controller.load(
+      model.detectorSource,
+      {
+        recognizerLarge: model.recognizerLarge,
+        recognizerMedium: model.recognizerMedium,
+        recognizerSmall: model.recognizerSmall,
+      },
+      model.language,
+      onDownloadProgressCallback
+    );
   }
 
-  static async forward(input: string) {
-    return await this.module.forward(input);
+  async forward(imageSource: string) {
+    return await this.controller.forward(imageSource);
   }
 
-  static onDownloadProgress(callback: (downloadProgress: number) => void) {
-    this.onDownloadProgressCallback = callback;
+  delete() {
+    this.controller.delete();
   }
 }

package/src/modules/computer_vision/ObjectDetectionModule.ts

@@ -1,17 +1,30 @@
-import { ObjectDetectionNativeModule } from '../../native/RnExecutorchModules';
+import { ResourceFetcher } from '../../utils/ResourceFetcher';
 import { ResourceSource } from '../../types/common';
+import { Detection } from '../../types/objectDetection';
+import { ETError, getError } from '../../Error';
 import { BaseModule } from '../BaseModule';
 
 export class ObjectDetectionModule extends BaseModule {
-  protected static override nativeModule = ObjectDetectionNativeModule;
-
-  static override async load(modelSource: ResourceSource) {
-    return await super.load(modelSource);
+  async load(
+    model: { modelSource: ResourceSource },
+    onDownloadProgressCallback: (progress: number) => void = () => {}
+  ): Promise<void> {
+    const paths = await ResourceFetcher.fetch(
+      onDownloadProgressCallback,
+      model.modelSource
+    );
+    if (paths === null || paths.length < 1) {
+      throw new Error('Download interrupted.');
+    }
+    this.nativeModule = global.loadObjectDetection(paths[0] || '');
   }
 
-  static override async forward(
-    input: string
-  ): ReturnType<typeof this.nativeModule.forward> {
-    return await this.nativeModule.forward(input);
+  async forward(
+    imageSource: string,
+    detectionThreshold: number = 0.7
+  ): Promise<Detection[]> {
+    if (this.nativeModule == null)
+      throw new Error(getError(ETError.ModuleNotLoaded));
+    return await this.nativeModule.generate(imageSource, detectionThreshold);
   }
 }

package/src/modules/computer_vision/StyleTransferModule.ts

@@ -1,17 +1,26 @@
-import { StyleTransferNativeModule } from '../../native/RnExecutorchModules';
+import { ResourceFetcher } from '../../utils/ResourceFetcher';
 import { ResourceSource } from '../../types/common';
+import { ETError, getError } from '../../Error';
 import { BaseModule } from '../BaseModule';
 
 export class StyleTransferModule extends BaseModule {
-  protected static override nativeModule = StyleTransferNativeModule;
-
-  static override async load(modelSource: ResourceSource) {
-    return await super.load(modelSource);
+  async load(
+    model: { modelSource: ResourceSource },
+    onDownloadProgressCallback: (progress: number) => void = () => {}
+  ): Promise<void> {
+    const paths = await ResourceFetcher.fetch(
+      onDownloadProgressCallback,
+      model.modelSource
+    );
+    if (paths === null || paths.length < 1) {
+      throw new Error('Download interrupted.');
+    }
+    this.nativeModule = global.loadStyleTransfer(paths[0] || '');
   }
 
-  static override async forward(
-    input: string
-  ): ReturnType<typeof this.nativeModule.forward> {
-    return await this.nativeModule.forward(input);
+  async forward(imageSource: string): Promise<string> {
+    if (this.nativeModule == null)
+      throw new Error(getError(ETError.ModuleNotLoaded));
+    return await this.nativeModule.generate(imageSource);
   }
 }

package/src/modules/computer_vision/TextToImageModule.ts

@@ -0,0 +1,93 @@
+import { ResourceFetcher } from '../../utils/ResourceFetcher';
+import { ResourceSource } from '../../types/common';
+import { BaseModule } from '../BaseModule';
+import { Buffer } from 'buffer';
+import { PNG } from 'pngjs/browser';
+
+export class TextToImageModule extends BaseModule {
+  private inferenceCallback: (stepIdx: number) => void;
+
+  constructor(inferenceCallback?: (stepIdx: number) => void) {
+    super();
+    this.inferenceCallback = (stepIdx: number) => {
+      inferenceCallback?.(stepIdx);
+    };
+  }
+
+  async load(
+    model: {
+      tokenizerSource: ResourceSource;
+      schedulerSource: ResourceSource;
+      encoderSource: ResourceSource;
+      unetSource: ResourceSource;
+      decoderSource: ResourceSource;
+    },
+    onDownloadProgressCallback: (progress: number) => void = () => {}
+  ): Promise<void> {
+    const results = await ResourceFetcher.fetch(
+      onDownloadProgressCallback,
+      model.tokenizerSource,
+      model.schedulerSource,
+      model.encoderSource,
+      model.unetSource,
+      model.decoderSource
+    );
+    if (!results) {
+      throw new Error('Failed to fetch one or more resources.');
+    }
+    const [tokenizerPath, schedulerPath, encoderPath, unetPath, decoderPath] =
+      results;
+
+    if (
+      !tokenizerPath ||
+      !schedulerPath ||
+      !encoderPath ||
+      !unetPath ||
+      !decoderPath
+    ) {
+      throw new Error('Download interrupted.');
+    }
+
+    const response = await fetch('file://' + schedulerPath);
+    const schedulerConfig = await response.json();
+
+    this.nativeModule = global.loadTextToImage(
+      tokenizerPath,
+      encoderPath,
+      unetPath,
+      decoderPath,
+      schedulerConfig.beta_start,
+      schedulerConfig.beta_end,
+      schedulerConfig.num_train_timesteps,
+      schedulerConfig.steps_offset
+    );
+  }
+
+  async forward(
+    input: string,
+    imageSize: number = 512,
+    numSteps: number = 5,
+    seed?: number
+  ): Promise<string> {
+    const output = await this.nativeModule.generate(
+      input,
+      imageSize,
+      numSteps,
+      seed ? seed : -1,
+      this.inferenceCallback
+    );
+    const outputArray = new Uint8Array(output);
+    if (!outputArray.length) {
+      return '';
+    }
+    const png = new PNG({ width: imageSize, height: imageSize });
+    png.data = Buffer.from(outputArray);
+    const pngBuffer = PNG.sync.write(png, { colorType: 6 });
+    const pngString = pngBuffer.toString('base64');
+    return pngString;
+  }
+
+  public interrupt(): void {
+    this.nativeModule.interrupt();
+  }
+}

package/src/modules/computer_vision/VerticalOCRModule.ts

@@ -3,39 +3,43 @@ import { ResourceSource } from '../../types/common';
 import { OCRLanguage } from '../../types/ocr';
 
 export class VerticalOCRModule {
-  static module: VerticalOCRController;
+  private controller: VerticalOCRController;
 
-  static onDownloadProgressCallback = (_downloadProgress: number) => {};
+  constructor() {
+    this.controller = new VerticalOCRController();
+  }
 
-  static async load(
-    detectorSources: {
+  async load(
+    model: {
       detectorLarge: ResourceSource;
       detectorNarrow: ResourceSource;
-    },
-    recognizerSources: {
       recognizerLarge: ResourceSource;
       recognizerSmall: ResourceSource;
+      language: OCRLanguage;
     },
-    language: OCRLanguage = 'en',
-    independentCharacters: boolean = false
+    independentCharacters: boolean,
+    onDownloadProgressCallback: (progress: number) => void = () => {}
   ) {
-    this.module = new VerticalOCRController({
-      modelDownloadProgressCallback: this.onDownloadProgressCallback,
-    });
-
-    await this.module.loadModel(
-      detectorSources,
-      recognizerSources,
-      language,
-      independentCharacters
+    await this.controller.load(
+      {
+        detectorLarge: model.detectorLarge,
+        detectorNarrow: model.detectorNarrow,
+      },
+      {
+        recognizerLarge: model.recognizerLarge,
+        recognizerSmall: model.recognizerSmall,
+      },
+      model.language,
+      independentCharacters,
+      onDownloadProgressCallback
     );
   }
 
-  static async forward(input: string) {
-    return await this.module.forward(input);
+  async forward(imageSource: string) {
+    return await this.controller.forward(imageSource);
   }
 
-  static onDownloadProgress(callback: (downloadProgress: number) => void) {
-    this.onDownloadProgressCallback = callback;
+  delete() {
+    this.controller.delete();
   }
 }

package/src/modules/general/ExecutorchModule.ts

@@ -1,54 +1,24 @@
-import { ETError, getError } from '../../Error';
-import { ETModuleNativeModule } from '../../native/RnExecutorchModules';
-import { ResourceSource } from '../../types/common';
-import { ETInput } from '../../types/common';
-import { getTypeIdentifier } from '../../types/common';
+import { TensorPtr } from '../../types/common';
 import { BaseModule } from '../BaseModule';
+import { ResourceSource } from '../../types/common';
+import { ResourceFetcher } from '../../utils/ResourceFetcher';
 
 export class ExecutorchModule extends BaseModule {
-  protected static override nativeModule = ETModuleNativeModule;
-
-  static override async load(modelSource: ResourceSource) {
-    return await super.load(modelSource);
-  }
-
-  static override async forward(input: ETInput[] | ETInput, shape: number[][]) {
-    if (!Array.isArray(input)) {
-      input = [input];
-    }
-
-    const inputTypeIdentifiers: number[] = [];
-    const modelInputs: number[][] = [];
-
-    for (const subInput of input) {
-      const typeIdentifier = getTypeIdentifier(subInput);
-      if (typeIdentifier === -1) {
-        throw new Error(getError(ETError.InvalidArgument));
-      }
-      inputTypeIdentifiers.push(typeIdentifier);
-      modelInputs.push(Array.from(subInput, (x: number | BigInt) => Number(x)));
-    }
-
-    try {
-      return await this.nativeModule.forward(
-        modelInputs,
-        shape,
-        inputTypeIdentifiers
-      );
-    } catch (e) {
-      throw new Error(getError(e));
-    }
-  }
-
-  static async loadMethod(methodName: string) {
-    try {
-      await this.nativeModule.loadMethod(methodName);
-    } catch (e) {
-      throw new Error(getError(e));
+  async load(
+    modelSource: ResourceSource,
+    onDownloadProgressCallback: (progress: number) => void = () => {}
+  ): Promise<void> {
+    const paths = await ResourceFetcher.fetch(
+      onDownloadProgressCallback,
+      modelSource
+    );
+    if (paths === null || paths.length < 1) {
+      throw new Error('Download interrupted.');
     }
+    this.nativeModule = global.loadExecutorchModule(paths[0] || '');
   }
 
-  static async loadForward() {
-    await this.loadMethod('forward');
+  async forward(inputTensor: TensorPtr[]): Promise<TensorPtr[]> {
+    return await this.forwardET(inputTensor);
   }
 }

package/src/modules/natural_language_processing/LLMModule.ts

@@ -1,41 +1,47 @@
 import { LLMController } from '../../controllers/LLMController';
 import { ResourceSource } from '../../types/common';
-import { ChatConfig, LLMTool, Message, ToolsConfig } from '../../types/llm';
+import {
+  ChatConfig,
+  GenerationConfig,
+  LLMTool,
+  Message,
+  ToolsConfig,
+} from '../../types/llm';
 
 export class LLMModule {
-  static controller: LLMController;
+  private controller: LLMController;
 
-  static async load({
-    modelSource,
-    tokenizerSource,
-    tokenizerConfigSource,
-    onDownloadProgressCallback,
+  constructor({
     tokenCallback,
     responseCallback,
     messageHistoryCallback,
   }: {
-    modelSource: ResourceSource;
-    tokenizerSource: ResourceSource;
-    tokenizerConfigSource: ResourceSource;
-    onDownloadProgressCallback?: (_downloadProgress: number) => void;
     tokenCallback?: (token: string) => void;
     responseCallback?: (response: string) => void;
     messageHistoryCallback?: (messageHistory: Message[]) => void;
-  }) {
+  } = {}) {
     this.controller = new LLMController({
-      tokenCallback: tokenCallback,
-      responseCallback: responseCallback,
-      messageHistoryCallback: messageHistoryCallback,
-      onDownloadProgressCallback: onDownloadProgressCallback,
+      tokenCallback,
+      responseCallback,
+      messageHistoryCallback,
     });
+  }
+
+  async load(
+    model: {
+      modelSource: ResourceSource;
+      tokenizerSource: ResourceSource;
+      tokenizerConfigSource: ResourceSource;
+    },
+    onDownloadProgressCallback: (progress: number) => void = () => {}
+  ) {
     await this.controller.load({
-      modelSource,
-      tokenizerSource,
-      tokenizerConfigSource,
+      ...model,
+      onDownloadProgressCallback,
     });
   }
 
-  static setTokenCallback({
+  setTokenCallback({
     tokenCallback,
   }: {
     tokenCallback: (token: string) => void;
@@ -43,44 +49,47 @@ export class LLMModule {
     this.controller.setTokenCallback(tokenCallback);
   }
 
-  static configure({
+  configure({
     chatConfig,
     toolsConfig,
+    generationConfig,
   }: {
     chatConfig?: Partial<ChatConfig>;
     toolsConfig?: ToolsConfig;
+    generationConfig?: GenerationConfig;
   }) {
-    this.controller.configure({ chatConfig, toolsConfig });
+    this.controller.configure({ chatConfig, toolsConfig, generationConfig });
   }
 
-  static async forward(input: string): Promise<string> {
+  async forward(input: string): Promise<string> {
     await this.controller.forward(input);
     return this.controller.response;
   }
 
-  static async generate(
-    messages: Message[],
-    tools?: LLMTool[]
-  ): Promise<string> {
+  async generate(messages: Message[], tools?: LLMTool[]): Promise<string> {
     await this.controller.generate(messages, tools);
     return this.controller.response;
   }
 
-  static async sendMessage(message: string): Promise<Message[]> {
+  async sendMessage(message: string): Promise<Message[]> {
     await this.controller.sendMessage(message);
     return this.controller.messageHistory;
   }
 
-  static async deleteMessage(index: number): Promise<Message[]> {
-    await this.controller.deleteMessage(index);
+  deleteMessage(index: number): Message[] {
+    this.controller.deleteMessage(index);
     return this.controller.messageHistory;
   }
 
-  static interrupt() {
+  interrupt() {
     this.controller.interrupt();
   }
 
-  static delete() {
+  getGeneratedTokenCount() {
+    return this.controller.getGeneratedTokenCount();
+  }
+
+  delete() {
     this.controller.delete();
   }
 }