react-native-executorch 0.3.2 → 0.4.1

package/src/types/common.ts

@@ -1,11 +1,3 @@
-import {
-  _ClassificationModule,
-  _StyleTransferModule,
-  _ObjectDetectionModule,
-  ETModule,
-  _ETModule,
-} from '../native/RnExecutorchModules';
-
 export const getTypeIdentifier = (input: ETInput): number => {
   if (input instanceof Int8Array) return 1;
   if (input instanceof Int32Array) return 3;
@@ -15,19 +7,7 @@ export const getTypeIdentifier = (input: ETInput): number => {
   return -1;
 };
 
-export type ResourceSource = string | number;
-
-export interface Model {
-  generate: (input: string) => Promise<void>;
-  response: string;
-  downloadProgress: number;
-  error: string | null;
-  isModelGenerating: boolean;
-  isGenerating: boolean;
-  isModelReady: boolean;
-  isReady: boolean;
-  interrupt: () => void;
-}
+export type ResourceSource = string | number | object;
 
 export type ETInput =
   | Int8Array
@@ -35,26 +15,3 @@ export type ETInput =
   | BigInt64Array
   | Float32Array
   | Float64Array;
-
-export interface ExecutorchModule {
-  error: string | null;
-  isReady: boolean;
-  isGenerating: boolean;
-  forward: (
-    inputs: ETInput[] | ETInput,
-    shapes: number[][]
-  ) => ReturnType<_ETModule['forward']>;
-  loadMethod: (methodName: string) => Promise<void>;
-  loadForward: () => Promise<void>;
-}
-
-export type Module =
-  | _ClassificationModule
-  | _StyleTransferModule
-  | _ObjectDetectionModule
-  | typeof ETModule;
-
-export interface MessageType {
-  role: 'user' | 'assistant';
-  content: string;
-}

package/src/types/imageSegmentation.ts

@@ -0,0 +1,25 @@
+/* eslint-disable @cspell/spellchecker */
+export enum DeeplabLabel {
+  BACKGROUND,
+  AEROPLANE,
+  BICYCLE,
+  BIRD,
+  BOAT,
+  BOTTLE,
+  BUS,
+  CAR,
+  CAT,
+  CHAIR,
+  COW,
+  DININGTABLE,
+  DOG,
+  HORSE,
+  MOTORBIKE,
+  PERSON,
+  POTTEDPLANT,
+  SHEEP,
+  SOFA,
+  TRAIN,
+  TVMONITOR,
+  ARGMAX, // Additional label not present in the model
+}

package/src/types/llm.ts

@@ -0,0 +1,57 @@
+export interface LLMType {
+  messageHistory: Message[];
+  response: string;
+  isReady: boolean;
+  isGenerating: boolean;
+  downloadProgress: number;
+  error: string | null;
+  configure: ({
+    chatConfig,
+    toolsConfig,
+  }: {
+    chatConfig?: Partial<ChatConfig>;
+    toolsConfig?: ToolsConfig;
+  }) => void;
+  generate: (messages: Message[], tools?: LLMTool[]) => Promise<void>;
+  sendMessage: (message: string) => Promise<void>;
+  deleteMessage: (index: number) => void;
+  interrupt: () => void;
+}
+
+export type MessageRole = 'user' | 'assistant' | 'system';
+
+export interface Message {
+  role: MessageRole;
+  content: string;
+}
+
+export interface ToolCall {
+  toolName: string;
+  arguments: Object;
+}
+
+// usually tool is represented with dictionary (Object), but fields depend on the model
+// unfortunately there's no one standard so it's hard to type it better
+export type LLMTool = Object;
+
+export interface ChatConfig {
+  initialMessageHistory: Message[];
+  contextWindowLength: number;
+  systemPrompt: string;
+}
+
+export interface ToolsConfig {
+  tools: LLMTool[];
+  executeToolCallback: (call: ToolCall) => Promise<string | null>;
+  displayToolCalls?: boolean;
+}
+
+export const SPECIAL_TOKENS = [
+  'bos_token',
+  'eos_token',
+  'unk_token',
+  'sep_token',
+  'pad_token',
+  'cls_token',
+  'mask_token',
+];

package/src/types/ocr.ts

@@ -1,3 +1,5 @@
+import { symbols } from '../constants/ocr/symbols';
+
 export interface OCRDetection {
   bbox: OCRBbox[];
   text: string;
@@ -9,4 +11,4 @@ export interface OCRBbox {
   y: number;
 }
 
-export type OCRLanguage = 'en';
+export type OCRLanguage = keyof typeof symbols;

package/src/types/stt.ts

@@ -0,0 +1,93 @@
+export interface ModelConfig {
+  sources: {
+    encoder: string;
+    decoder: string;
+  };
+  tokenizer: {
+    source: string;
+    bos: number;
+    eos: number;
+  };
+  isMultilingual: boolean;
+}
+
+// Those languages are supported just by whisper multilingual
+export enum SpeechToTextLanguage {
+  Afrikaans = 'af',
+  Albanian = 'sq',
+  Arabic = 'ar',
+  Armenian = 'hy',
+  Azerbaijani = 'az',
+  Basque = 'eu',
+  Belarusian = 'be',
+  Bengali = 'bn',
+  Bosnian = 'bs',
+  Bulgarian = 'bg',
+  Burmese = 'my',
+  Catalan = 'ca',
+  Chinese = 'zh',
+  Croatian = 'hr',
+  Czech = 'cs',
+  Danish = 'da',
+  Dutch = 'nl',
+  Estonian = 'et',
+  English = 'en',
+  Finnish = 'fi',
+  French = 'fr',
+  Galician = 'gl',
+  Georgian = 'ka',
+  German = 'de',
+  Greek = 'el',
+  Gujarati = 'gu',
+  HaitianCreole = 'ht',
+  Hebrew = 'he',
+  Hindi = 'hi',
+  Hungarian = 'hu',
+  Icelandic = 'is',
+  Indonesian = 'id',
+  Italian = 'it',
+  Japanese = 'ja',
+  Kannada = 'kn',
+  Kazakh = 'kk',
+  Khmer = 'km',
+  Korean = 'ko',
+  Lao = 'lo',
+  Latvian = 'lv',
+  Lithuanian = 'lt',
+  Macedonian = 'mk',
+  Malagasy = 'mg',
+  Malay = 'ms',
+  Malayalam = 'ml',
+  Maltese = 'mt',
+  Marathi = 'mr',
+  Nepali = 'ne',
+  Norwegian = 'no',
+  Persian = 'fa',
+  Polish = 'pl',
+  Portuguese = 'pt',
+  Punjabi = 'pa',
+  Romanian = 'ro',
+  Russian = 'ru',
+  Serbian = 'sr',
+  Sinhala = 'si',
+  Slovak = 'sk',
+  Slovenian = 'sl',
+  Spanish = 'es',
+  Sundanese = 'su',
+  Swahili = 'sw',
+  Swedish = 'sv',
+  Tagalog = 'tl',
+  Tajik = 'tg',
+  Tamil = 'ta',
+  Telugu = 'te',
+  Thai = 'th',
+  Turkish = 'tr',
+  Ukrainian = 'uk',
+  Urdu = 'ur',
+  Uzbek = 'uz',
+  Vietnamese = 'vi',
+  Welsh = 'cy',
+  Yiddish = 'yi',
+}
+
+export type AvailableModels = 'whisper' | 'moonshine' | 'whisperMultilingual';

package/src/utils/ResourceFetcher.ts

@@ -0,0 +1,196 @@
+import {
+  cacheDirectory,
+  createDownloadResumable,
+  getInfoAsync,
+  makeDirectoryAsync,
+  moveAsync,
+  FileSystemSessionType,
+  writeAsStringAsync,
+  EncodingType,
+  deleteAsync,
+  readDirectoryAsync,
+} from 'expo-file-system';
+import { Asset } from 'expo-asset';
+import { RNEDirectory } from '../constants/directories';
+import { ResourceSource } from '../types/common';
+
+export class ResourceFetcher {
+  static async fetch(
+    source: ResourceSource,
+    callback: (downloadProgress: number) => void = () => {}
+  ) {
+    if (typeof source === 'object') {
+      return this.handleObject(source);
+    }
+
+    const uri =
+      typeof source === 'number' ? Asset.fromModule(source).uri : source;
+
+    // Handle local files
+    if (uri.startsWith('file://')) {
+      return this.removeFilePrefix(uri);
+    }
+
+    const filename = this.getFilenameFromUri(uri);
+    const fileUri = `${RNEDirectory}${filename}`;
+
+    if (await this.checkFileExists(fileUri)) {
+      return this.removeFilePrefix(fileUri);
+    }
+    await this.createDirectoryIfNoExists();
+
+    // Handle local asset files in release mode
+    if (!uri.includes('://')) {
+      const asset = Asset.fromModule(source);
+      const fileUriWithType = `${fileUri}.${asset.type}`;
+      await asset.downloadAsync();
+      if (!asset.localUri) {
+        throw new Error(`Asset local URI is not available for ${source}`);
+      }
+      await moveAsync({ from: asset.localUri, to: fileUriWithType });
+      return this.removeFilePrefix(fileUriWithType);
+    }
+
+    // Handle remote file download
+    const cacheFileUri = `${cacheDirectory}${filename}`;
+    const downloadResumable = createDownloadResumable(
+      uri,
+      cacheFileUri,
+      { sessionType: FileSystemSessionType.BACKGROUND },
+      ({ totalBytesWritten, totalBytesExpectedToWrite }) => {
+        callback(totalBytesWritten / totalBytesExpectedToWrite);
+      }
+    );
+    const result = await downloadResumable.downloadAsync();
+    if (!result || result.status !== 200) {
+      throw new Error(`Failed to fetch resource from '${uri}'`);
+    }
+    await moveAsync({ from: cacheFileUri, to: fileUri });
+
+    this.triggerHuggingFaceDownloadCounter(uri);
+
+    return this.removeFilePrefix(fileUri);
+  }
+
+  static async fetchMultipleResources(
+    callback: (downloadProgress: number) => void = () => {},
+    ...sources: ResourceSource[]
+  ) {
+    const paths = [];
+
+    for (let idx = 0; idx < sources.length; idx++) {
+      paths.push(
+        await this.fetch(
+          sources[idx]!,
+          this.calculateDownloadProgress(sources.length, idx, callback)
+        )
+      );
+    }
+
+    return paths;
+  }
+
+  static async deleteMultipleResources(...sources: ResourceSource[]) {
+    for (const source of sources) {
+      const filename = this.getFilenameFromUri(source as string);
+      const fileUri = `${RNEDirectory}${filename}`;
+      if (await this.checkFileExists(fileUri)) {
+        await deleteAsync(fileUri);
+      }
+    }
+  }
+
+  private static calculateDownloadProgress(
+    numberOfFiles: number,
+    currentFileIndex: number,
+    setProgress: (downloadProgress: number) => void
+  ) {
+    return (progress: number) => {
+      if (progress === 1 && currentFileIndex === numberOfFiles - 1) {
+        setProgress(1);
+        return;
+      }
+      const contributionPerFile = 1 / numberOfFiles;
+      const baseProgress = contributionPerFile * currentFileIndex;
+      const scaledProgress = progress * contributionPerFile;
+      const updatedProgress = baseProgress + scaledProgress;
+      setProgress(updatedProgress);
+    };
+  }
+
+  static async listDownloadedFiles() {
+    const files = await readDirectoryAsync(RNEDirectory);
+    return files.map((file) => `${RNEDirectory}${file}`);
+  }
+
+  static async listDownloadedModels() {
+    const files = await this.listDownloadedFiles();
+    return files.filter((file) => file.endsWith('.pte'));
+  }
+
+  private static async handleObject(source: object) {
+    const jsonString = JSON.stringify(source);
+    const digest = this.hashObject(jsonString);
+    const filename = `${digest}.json`;
+    const path = `${RNEDirectory}${filename}`;
+
+    if (await this.checkFileExists(path)) {
+      return this.removeFilePrefix(path);
+    }
+
+    await this.createDirectoryIfNoExists();
+    await writeAsStringAsync(path, jsonString, {
+      encoding: EncodingType.UTF8,
+    });
+
+    return this.removeFilePrefix(path);
+  }
+
+  private static getFilenameFromUri(uri: string) {
+    let cleanUri = uri.replace(/^https?:\/\//, '');
+    cleanUri = cleanUri.split('?')?.[0]?.split('#')?.[0] ?? cleanUri;
+    return cleanUri.replace(/[^a-zA-Z0-9._-]/g, '_');
+  }
+
+  private static removeFilePrefix(uri: string) {
+    return uri.startsWith('file://') ? uri.slice(7) : uri;
+  }
+
+  private static hashObject(jsonString: string) {
+    let hash = 0;
+    for (let i = 0; i < jsonString.length; i++) {
+      // eslint-disable-next-line no-bitwise
+      hash = (hash << 5) - hash + jsonString.charCodeAt(i);
+      // eslint-disable-next-line no-bitwise
+      hash |= 0;
+    }
+    // eslint-disable-next-line no-bitwise
+    return (hash >>> 0).toString();
+  }
+
+  /*
+   * Increments the Hugging Face download counter if the URI points to a Software Mansion Hugging Face repo.
+   * More information: https://huggingface.co/docs/hub/models-download-stats
+   */
+  private static triggerHuggingFaceDownloadCounter(uri: string) {
+    const url = new URL(uri);
+    if (
+      url.host === 'huggingface.co' &&
+      url.pathname.startsWith('/software-mansion/')
+    ) {
+      const baseUrl = `${url.protocol}//${url.host}${url.pathname.split('resolve')[0]}`;
+      fetch(`${baseUrl}resolve/main/config.json`, { method: 'HEAD' });
+    }
+  }
+
+  private static async createDirectoryIfNoExists() {
+    if (!(await this.checkFileExists(RNEDirectory))) {
+      await makeDirectoryAsync(RNEDirectory, { intermediates: true });
+    }
+  }
+
+  private static async checkFileExists(fileUri: string) {
+    const fileInfo = await getInfoAsync(fileUri);
+    return fileInfo.exists;
+  }
+}

package/src/utils/llm.ts

@@ -0,0 +1,34 @@
+import { LLMTool, ToolCall } from '../types/llm';
+
+export const parseToolCall: (message: string) => ToolCall[] = (
+  message: string
+) => {
+  try {
+    const unparsedToolCalls = message.match('\\[(.|\\s)*\\]');
+    if (!unparsedToolCalls) {
+      throw Error('Regex did not match array.');
+    }
+    const parsedMessage: LLMTool[] = JSON.parse(unparsedToolCalls[0]);
+    const results = [];
+
+    for (const tool of parsedMessage) {
+      if (
+        'name' in tool &&
+        typeof tool.name === 'string' &&
+        'arguments' in tool &&
+        tool.arguments !== null &&
+        typeof tool.arguments === 'object'
+      ) {
+        results.push({
+          toolName: tool.name,
+          arguments: tool.arguments,
+        });
+      }
+    }
+
+    return results;
+  } catch (e) {
+    console.error(e);
+    return [];
+  }
+};

package/src/utils/stt.ts

@@ -0,0 +1,28 @@
+export const longCommonInfPref = (
+  seq1: number[],
+  seq2: number[],
+  hammingDistThreshold: number
+) => {
+  let maxInd = 0;
+  let maxLength = 0;
+
+  for (let i = 0; i < seq1.length; i++) {
+    let j = 0;
+    let hammingDist = 0;
+    while (
+      j < seq2.length &&
+      i + j < seq1.length &&
+      (seq1[i + j] === seq2[j] || hammingDist < hammingDistThreshold)
+    ) {
+      if (seq1[i + j] !== seq2[j]) {
+        hammingDist++;
+      }
+      j++;
+    }
+    if (j >= maxLength) {
+      maxLength = j;
+      maxInd = i;
+    }
+  }
+  return maxInd;
+};

package/android/src/main/java/com/swmansion/rnexecutorch/utils/llms/ConversationManager.kt

@@ -1,68 +0,0 @@
-package com.swmansion.rnexecutorch.utils.llms
-
-enum class ChatRole {
-  SYSTEM,
-  USER,
-  ASSISTANT,
-}
-
-const val BEGIN_OF_TEXT_TOKEN = "<|begin_of_text|>"
-const val END_OF_TEXT_TOKEN = "<|eot_id|>"
-const val START_HEADER_ID_TOKEN = "<|start_header_id|>"
-const val END_HEADER_ID_TOKEN = "<|end_header_id|>"
-
-class ConversationManager(
-  private val numMessagesContextWindow: Int,
-  systemPrompt: String,
-  messageHistory: Array<Map<String, String>>,
-) {
-  private val basePrompt: String
-  private val messages = ArrayDeque<String>()
-
-  init {
-    this.basePrompt =
-      BEGIN_OF_TEXT_TOKEN +
-      getHeaderTokenFromRole(ChatRole.SYSTEM) +
-      systemPrompt +
-      END_OF_TEXT_TOKEN +
-      getHeaderTokenFromRole(ChatRole.USER)
-
-    messageHistory.forEach { message ->
-      when (message["role"]) {
-        "user" -> addResponse(message["content"]!!, ChatRole.USER)
-        "assistant" -> addResponse(message["content"]!!, ChatRole.ASSISTANT)
-      }
-    }
-  }
-
-  fun addResponse(
-    text: String,
-    senderRole: ChatRole,
-  ) {
-    if (this.messages.size >= this.numMessagesContextWindow) {
-      this.messages.removeFirst()
-    }
-    val formattedMessage: String =
-      if (senderRole == ChatRole.ASSISTANT) {
-        text + getHeaderTokenFromRole(ChatRole.USER)
-      } else {
-        text + END_OF_TEXT_TOKEN + getHeaderTokenFromRole(ChatRole.ASSISTANT)
-      }
-    this.messages.add(formattedMessage)
-  }
-
-  fun getConversation(): String {
-    val prompt = StringBuilder(this.basePrompt)
-    for (elem in this.messages) {
-      prompt.append(elem)
-    }
-    return prompt.toString()
-  }
-
-  private fun getHeaderTokenFromRole(role: ChatRole): String =
-    when (role) {
-      ChatRole.SYSTEM -> START_HEADER_ID_TOKEN + "system" + END_HEADER_ID_TOKEN
-      ChatRole.USER -> START_HEADER_ID_TOKEN + "user" + END_HEADER_ID_TOKEN
-      ChatRole.ASSISTANT -> START_HEADER_ID_TOKEN + "assistant" + END_HEADER_ID_TOKEN
-    }
-}

package/ios/ExecutorchLib.xcframework/ios-arm64-simulator/ExecutorchLib.framework/_CodeSignature/CodeResources

@@ -1,124 +0,0 @@
-<?xml version="1.0" encoding="UTF-8"?>
-<!DOCTYPE plist PUBLIC "-//Apple//DTD PLIST 1.0//EN" "http://www.apple.com/DTDs/PropertyList-1.0.dtd">
-<plist version="1.0">
-<dict>
-	<key>files</key>
-	<dict>
-		<key>Headers/ETModel.h</key>
-		<data>
-		CFAz750OjepOG7MVBPABGfKHNeI=
-		</data>
-		<key>Headers/LLaMARunner.h</key>
-		<data>
-		SU8Fo2gR+gVVl9IplHgBJBRh1gQ=
-		</data>
-		<key>Info.plist</key>
-		<data>
-		l3rE2nBARVh++WIyHCfeHXD6Ewo=
-		</data>
-	</dict>
-	<key>files2</key>
-	<dict>
-		<key>Headers/ETModel.h</key>
-		<dict>
-			<key>hash2</key>
-			<data>
-			UXFd6a5OARqV4JnB+Jm4uqmt15aUmnXSOLPQKZTWZCc=
-			</data>
-		</dict>
-		<key>Headers/LLaMARunner.h</key>
-		<dict>
-			<key>hash2</key>
-			<data>
-			or8gFkCO2QVkQgaeFAaqs/WqGjv8kABL8Rzcdcuexw0=
-			</data>
-		</dict>
-	</dict>
-	<key>rules</key>
-	<dict>
-		<key>^.*</key>
-		<true/>
-		<key>^.*\.lproj/</key>
-		<dict>
-			<key>optional</key>
-			<true/>
-			<key>weight</key>
-			<real>1000</real>
-		</dict>
-		<key>^.*\.lproj/locversion.plist$</key>
-		<dict>
-			<key>omit</key>
-			<true/>
-			<key>weight</key>
-			<real>1100</real>
-		</dict>
-		<key>^Base\.lproj/</key>
-		<dict>
-			<key>weight</key>
-			<real>1010</real>
-		</dict>
-		<key>^version.plist$</key>
-		<true/>
-	</dict>
-	<key>rules2</key>
-	<dict>
-		<key>.*\.dSYM($|/)</key>
-		<dict>
-			<key>weight</key>
-			<real>11</real>
-		</dict>
-		<key>^(.*/)?\.DS_Store$</key>
-		<dict>
-			<key>omit</key>
-			<true/>
-			<key>weight</key>
-			<real>2000</real>
-		</dict>
-		<key>^.*</key>
-		<true/>
-		<key>^.*\.lproj/</key>
-		<dict>
-			<key>optional</key>
-			<true/>
-			<key>weight</key>
-			<real>1000</real>
-		</dict>
-		<key>^.*\.lproj/locversion.plist$</key>
-		<dict>
-			<key>omit</key>
-			<true/>
-			<key>weight</key>
-			<real>1100</real>
-		</dict>
-		<key>^Base\.lproj/</key>
-		<dict>
-			<key>weight</key>
-			<real>1010</real>
-		</dict>
-		<key>^Info\.plist$</key>
-		<dict>
-			<key>omit</key>
-			<true/>
-			<key>weight</key>
-			<real>20</real>
-		</dict>
-		<key>^PkgInfo$</key>
-		<dict>
-			<key>omit</key>
-			<true/>
-			<key>weight</key>
-			<real>20</real>
-		</dict>
-		<key>^embedded\.provisionprofile$</key>
-		<dict>
-			<key>weight</key>
-			<real>20</real>
-		</dict>
-		<key>^version\.plist$</key>
-		<dict>
-			<key>weight</key>
-			<real>20</real>
-		</dict>
-	</dict>
-</dict>
-</plist>

package/ios/RnExecutorch/utils/llms/Constants.h

@@ -1,6 +0,0 @@
-#import <Foundation/Foundation.h>
-
-extern NSString *const END_OF_TEXT_TOKEN_NS;
-extern NSString *const BEGIN_OF_TEXT_TOKEN_NS;
-extern NSString *const START_HEADER_ID_TOKEN_NS;
-extern NSString *const END_HEADER_ID_TOKEN_NS;