react-native-executorch 0.5.15 → 0.6.0

package/src/modules/computer_vision/TextToImageModule.ts

@@ -0,0 +1,93 @@
+import { ResourceFetcher } from '../../utils/ResourceFetcher';
+import { ResourceSource } from '../../types/common';
+import { BaseModule } from '../BaseModule';
+import { Buffer } from 'buffer';
+import { PNG } from 'pngjs/browser';
+
+export class TextToImageModule extends BaseModule {
+  private inferenceCallback: (stepIdx: number) => void;
+
+  constructor(inferenceCallback?: (stepIdx: number) => void) {
+    super();
+    this.inferenceCallback = (stepIdx: number) => {
+      inferenceCallback?.(stepIdx);
+    };
+  }
+
+  async load(
+    model: {
+      tokenizerSource: ResourceSource;
+      schedulerSource: ResourceSource;
+      encoderSource: ResourceSource;
+      unetSource: ResourceSource;
+      decoderSource: ResourceSource;
+    },
+    onDownloadProgressCallback: (progress: number) => void = () => {}
+  ): Promise<void> {
+    const results = await ResourceFetcher.fetch(
+      onDownloadProgressCallback,
+      model.tokenizerSource,
+      model.schedulerSource,
+      model.encoderSource,
+      model.unetSource,
+      model.decoderSource
+    );
+    if (!results) {
+      throw new Error('Failed to fetch one or more resources.');
+    }
+    const [tokenizerPath, schedulerPath, encoderPath, unetPath, decoderPath] =
+      results;
+
+    if (
+      !tokenizerPath ||
+      !schedulerPath ||
+      !encoderPath ||
+      !unetPath ||
+      !decoderPath
+    ) {
+      throw new Error('Download interrupted.');
+    }
+
+    const response = await fetch('file://' + schedulerPath);
+    const schedulerConfig = await response.json();
+
+    this.nativeModule = global.loadTextToImage(
+      tokenizerPath,
+      encoderPath,
+      unetPath,
+      decoderPath,
+      schedulerConfig.beta_start,
+      schedulerConfig.beta_end,
+      schedulerConfig.num_train_timesteps,
+      schedulerConfig.steps_offset
+    );
+  }
+
+  async forward(
+    input: string,
+    imageSize: number = 512,
+    numSteps: number = 5,
+    seed?: number
+  ): Promise<string> {
+    const output = await this.nativeModule.generate(
+      input,
+      imageSize,
+      numSteps,
+      seed ? seed : -1,
+      this.inferenceCallback
+    );
+    const outputArray = new Uint8Array(output);
+    if (!outputArray.length) {
+      return '';
+    }
+    const png = new PNG({ width: imageSize, height: imageSize });
+    png.data = Buffer.from(outputArray);
+    const pngBuffer = PNG.sync.write(png, { colorType: 6 });
+    const pngString = pngBuffer.toString('base64');
+    return pngString;
+  }
+
+  public interrupt(): void {
+    this.nativeModule.interrupt();
+  }
+}

package/src/modules/computer_vision/VerticalOCRModule.ts

@@ -35,8 +35,8 @@ export class VerticalOCRModule {
     );
   }
 
-  async forward(input: string) {
-    return await this.controller.forward(input);
+  async forward(imageSource: string) {
+    return await this.controller.forward(imageSource);
   }
 
   delete() {

package/src/modules/natural_language_processing/SpeechToTextModule.ts

@@ -43,6 +43,10 @@ export class SpeechToTextModule {
     );
   }
 
+  public delete(): void {
+    this.nativeModule.unload();
+  }
+
   public async encode(
     waveform: Float32Array | number[]
   ): Promise<Float32Array> {
@@ -150,18 +154,18 @@ export class SpeechToTextModule {
     }
   }
 
-  public async streamInsert(waveform: Float32Array | number[]): Promise<void> {
+  public streamInsert(waveform: Float32Array | number[]): void {
     if (Array.isArray(waveform)) {
       Logger.info(
         'Passing waveform as number[] is deprecated, use Float32Array instead'
       );
       waveform = new Float32Array(waveform);
     }
-    return this.nativeModule.streamInsert(waveform);
+    this.nativeModule.streamInsert(waveform);
   }
 
-  public async streamStop(): Promise<void> {
-    return this.nativeModule.streamStop();
+  public streamStop(): void {
+    this.nativeModule.streamStop();
   }
 
   private validateOptions(options: DecodingOptions) {

package/src/modules/natural_language_processing/VADModule.ts

@@ -0,0 +1,27 @@
+import { ResourceFetcher } from '../../utils/ResourceFetcher';
+import { ResourceSource } from '../../types/common';
+import { Segment } from '../../types/vad';
+import { ETError, getError } from '../../Error';
+import { BaseModule } from '../BaseModule';
+
+export class VADModule extends BaseModule {
+  async load(
+    model: { modelSource: ResourceSource },
+    onDownloadProgressCallback: (progress: number) => void = () => {}
+  ): Promise<void> {
+    const paths = await ResourceFetcher.fetch(
+      onDownloadProgressCallback,
+      model.modelSource
+    );
+    if (paths === null || paths.length < 1) {
+      throw new Error('Download interrupted.');
+    }
+    this.nativeModule = global.loadVAD(paths[0] || '');
+  }
+
+  async forward(waveform: Float32Array): Promise<Segment[]> {
+    if (this.nativeModule == null)
+      throw new Error(getError(ETError.ModuleNotLoaded));
+    return await this.nativeModule.generate(waveform);
+  }
+}

package/src/types/llm.ts

@@ -51,6 +51,8 @@ export interface ToolsConfig {
 }
 
 export interface GenerationConfig {
+  temperature?: number;
+  topp?: number;
   outputTokenBatchSize?: number;
   batchTimeInterval?: number;
 }

package/src/types/vad.ts

@@ -0,0 +1,4 @@
+export interface Segment {
+  start: number;
+  end: number;
+}

package/src/utils/ResourceFetcher.ts

@@ -27,6 +27,7 @@
  *   - Implements linked list behavior via the `.next` attribute
  *   - Automatically processes subsequent downloads when `.next` contains a valid resource
  */
+
 import {
   cacheDirectory,
   copyAsync,
@@ -37,7 +38,7 @@ import {
   EncodingType,
   deleteAsync,
   readDirectoryAsync,
-} from 'expo-file-system';
+} from 'expo-file-system/legacy';
 import { Asset } from 'expo-asset';
 import { Platform } from 'react-native';
 import { RNEDirectory } from '../constants/directories';

package/src/utils/ResourceFetcherUtils.ts

@@ -1,16 +1,16 @@
+import { RNEDirectory } from '../constants/directories';
+import { ResourceSource } from '../types/common';
+import { Asset } from 'expo-asset';
+import { Logger } from '../common/Logger';
+
 /**
  * @internal
  */
-
 import {
-  DownloadResumable,
   getInfoAsync,
   makeDirectoryAsync,
-} from 'expo-file-system';
-import { RNEDirectory } from '../constants/directories';
-import { ResourceSource } from '../types/common';
-import { Asset } from 'expo-asset';
-import { Logger } from '../common/Logger';
+  type DownloadResumable,
+} from 'expo-file-system/legacy';
 
 export const enum HTTP_CODE {
   OK = 200,
@@ -75,7 +75,7 @@ export namespace ResourceFetcherUtils {
     let totalLength = 0;
     let previousFilesTotalLength = 0;
     for (const source of sources) {
-      const type = await ResourceFetcherUtils.getType(source);
+      const type = ResourceFetcherUtils.getType(source);
       let length = 0;
       try {
         if (type === SourceType.REMOTE_FILE && typeof source === 'string') {

package/third-party/include/c10/macros/Export.h

@@ -1,79 +1 @@
-#define C10_USING_CUSTOM_GENERATED_MACROS
-#ifndef C10_MACROS_EXPORT_H_
-#define C10_MACROS_EXPORT_H_
-
-#ifndef C10_USING_CUSTOM_GENERATED_MACROS
-#include <c10/macros/cmake_macros.h>
-#endif // C10_USING_CUSTOM_GENERATED_MACROS
-
 #include <torch/headeronly/macros/Export.h>
-
-// This one is being used by libtorch.so
-#ifdef CAFFE2_BUILD_MAIN_LIB
-#define TORCH_API C10_EXPORT
-#else
-#define TORCH_API C10_IMPORT
-#endif
-
-// You may be wondering: Whose brilliant idea was it to split torch_cuda into
-// two pieces with confusing names?
-// Once upon a time, there _was_ only TORCH_CUDA_API. All was happy until we
-// tried to compile PyTorch for CUDA 11.1, which ran into relocation marker
-// issues when linking big binaries.
-// (https://github.com/pytorch/pytorch/issues/39968) We had two choices:
-//    (1) Stop supporting so many GPU architectures
-//    (2) Do something else
-// We chose #2 and decided to split the behemoth that was torch_cuda into two
-// smaller libraries, one with most of the core kernel functions (torch_cuda_cu)
-// and the other that had..well..everything else (torch_cuda_cpp). The idea was
-// this: instead of linking our static libraries (like the hefty
-// libcudnn_static.a) with another huge library, torch_cuda, and run into pesky
-// relocation marker issues, we could link our static libraries to a smaller
-// part of torch_cuda (torch_cuda_cpp) and avoid the issues.
-
-// libtorch_cuda_cu.so
-#ifdef TORCH_CUDA_CU_BUILD_MAIN_LIB
-#define TORCH_CUDA_CU_API C10_EXPORT
-#elif defined(BUILD_SPLIT_CUDA)
-#define TORCH_CUDA_CU_API C10_IMPORT
-#endif
-
-// libtorch_cuda_cpp.so
-#ifdef TORCH_CUDA_CPP_BUILD_MAIN_LIB
-#define TORCH_CUDA_CPP_API C10_EXPORT
-#elif defined(BUILD_SPLIT_CUDA)
-#define TORCH_CUDA_CPP_API C10_IMPORT
-#endif
-
-// libtorch_cuda.so (where torch_cuda_cu and torch_cuda_cpp are a part of the
-// same api)
-#ifdef TORCH_CUDA_BUILD_MAIN_LIB
-#define TORCH_CUDA_CPP_API C10_EXPORT
-#define TORCH_CUDA_CU_API C10_EXPORT
-#elif !defined(BUILD_SPLIT_CUDA)
-#define TORCH_CUDA_CPP_API C10_IMPORT
-#define TORCH_CUDA_CU_API C10_IMPORT
-#endif
-
-#if defined(TORCH_HIP_BUILD_MAIN_LIB)
-#define TORCH_HIP_CPP_API C10_EXPORT
-#define TORCH_HIP_API C10_EXPORT
-#else
-#define TORCH_HIP_CPP_API C10_IMPORT
-#define TORCH_HIP_API C10_IMPORT
-#endif
-
-#if defined(TORCH_XPU_BUILD_MAIN_LIB)
-#define TORCH_XPU_API C10_EXPORT
-#else
-#define TORCH_XPU_API C10_IMPORT
-#endif
-
-// Enums only need to be exported on windows for non-CUDA files
-#if defined(_WIN32) && defined(__CUDACC__)
-#define C10_API_ENUM C10_API
-#else
-#define C10_API_ENUM
-#endif
-
-#endif // C10_MACROS_EXPORT_H_