react-native-executorch 0.3.3 → 0.4.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (344) hide show
  1. package/README.md +30 -13
  2. package/android/build.gradle +1 -1
  3. package/android/libs/executorch.aar +0 -0
  4. package/android/src/main/java/com/swmansion/rnexecutorch/ETModule.kt +1 -2
  5. package/android/src/main/java/com/swmansion/rnexecutorch/ImageSegmentation.kt +58 -0
  6. package/android/src/main/java/com/swmansion/rnexecutorch/LLM.kt +13 -49
  7. package/android/src/main/java/com/swmansion/rnexecutorch/RnExecutorchPackage.kt +37 -0
  8. package/android/src/main/java/com/swmansion/rnexecutorch/StyleTransfer.kt +1 -1
  9. package/android/src/main/java/com/swmansion/rnexecutorch/TextEmbeddings.kt +51 -0
  10. package/android/src/main/java/com/swmansion/rnexecutorch/Tokenizer.kt +86 -0
  11. package/android/src/main/java/com/swmansion/rnexecutorch/models/BaseModel.kt +3 -4
  12. package/android/src/main/java/com/swmansion/rnexecutorch/models/TextEmbeddings/TextEmbeddingsModel.kt +48 -0
  13. package/android/src/main/java/com/swmansion/rnexecutorch/models/TextEmbeddings/TextEmbeddingsUtils.kt +37 -0
  14. package/android/src/main/java/com/swmansion/rnexecutorch/models/classification/ClassificationModel.kt +1 -0
  15. package/android/src/main/java/com/swmansion/rnexecutorch/models/imageSegmentation/Constants.kt +26 -0
  16. package/android/src/main/java/com/swmansion/rnexecutorch/models/imageSegmentation/ImageSegmentationModel.kt +142 -0
  17. package/android/src/main/java/com/swmansion/rnexecutorch/models/ocr/utils/RecognizerUtils.kt +3 -0
  18. package/android/src/main/java/com/swmansion/rnexecutorch/models/{StyleTransferModel.kt → styleTransfer/StyleTransferModel.kt} +2 -1
  19. package/android/src/main/java/com/swmansion/rnexecutorch/utils/ArrayUtils.kt +0 -8
  20. package/android/src/main/java/com/swmansion/rnexecutorch/{models/classification/Utils.kt → utils/Numerical.kt} +1 -1
  21. package/ios/ExecutorchLib.xcframework/Info.plist +4 -4
  22. package/ios/ExecutorchLib.xcframework/ios-arm64/ExecutorchLib.framework/ExecutorchLib +0 -0
  23. package/ios/ExecutorchLib.xcframework/ios-arm64/ExecutorchLib.framework/Headers/HuggingFaceTokenizer.h +14 -0
  24. package/ios/ExecutorchLib.xcframework/ios-arm64/ExecutorchLib.framework/Headers/LLaMARunner.h +1 -23
  25. package/ios/ExecutorchLib.xcframework/ios-arm64/ExecutorchLib.framework/Info.plist +0 -0
  26. package/ios/ExecutorchLib.xcframework/ios-arm64-simulator/ExecutorchLib.framework/ExecutorchLib +0 -0
  27. package/ios/ExecutorchLib.xcframework/ios-arm64-simulator/ExecutorchLib.framework/Headers/HuggingFaceTokenizer.h +14 -0
  28. package/ios/ExecutorchLib.xcframework/ios-arm64-simulator/ExecutorchLib.framework/Headers/LLaMARunner.h +1 -23
  29. package/ios/ExecutorchLib.xcframework/ios-arm64-simulator/ExecutorchLib.framework/Info.plist +0 -0
  30. package/ios/RnExecutorch/Classification.mm +15 -18
  31. package/ios/RnExecutorch/ETModule.mm +6 -5
  32. package/ios/RnExecutorch/ImageSegmentation.h +5 -0
  33. package/ios/RnExecutorch/ImageSegmentation.mm +60 -0
  34. package/ios/RnExecutorch/LLM.mm +12 -53
  35. package/ios/RnExecutorch/OCR.mm +39 -43
  36. package/ios/RnExecutorch/ObjectDetection.mm +20 -20
  37. package/ios/RnExecutorch/SpeechToText.mm +6 -7
  38. package/ios/RnExecutorch/StyleTransfer.mm +16 -19
  39. package/ios/RnExecutorch/TextEmbeddings.h +5 -0
  40. package/ios/RnExecutorch/TextEmbeddings.mm +62 -0
  41. package/ios/RnExecutorch/Tokenizer.h +5 -0
  42. package/ios/RnExecutorch/Tokenizer.mm +83 -0
  43. package/ios/RnExecutorch/VerticalOCR.mm +36 -36
  44. package/ios/RnExecutorch/models/BaseModel.h +2 -5
  45. package/ios/RnExecutorch/models/BaseModel.mm +5 -15
  46. package/ios/RnExecutorch/models/classification/ClassificationModel.mm +2 -3
  47. package/ios/RnExecutorch/models/classification/Constants.mm +0 -1
  48. package/ios/RnExecutorch/models/image_segmentation/Constants.h +4 -0
  49. package/ios/RnExecutorch/models/image_segmentation/Constants.mm +8 -0
  50. package/ios/RnExecutorch/models/image_segmentation/ImageSegmentationModel.h +10 -0
  51. package/ios/RnExecutorch/models/image_segmentation/ImageSegmentationModel.mm +146 -0
  52. package/ios/RnExecutorch/models/object_detection/SSDLiteLargeModel.mm +1 -2
  53. package/ios/RnExecutorch/models/ocr/Detector.h +0 -2
  54. package/ios/RnExecutorch/models/ocr/Detector.mm +2 -1
  55. package/ios/RnExecutorch/models/ocr/RecognitionHandler.h +5 -4
  56. package/ios/RnExecutorch/models/ocr/RecognitionHandler.mm +9 -26
  57. package/ios/RnExecutorch/models/ocr/Recognizer.mm +1 -2
  58. package/ios/RnExecutorch/models/ocr/VerticalDetector.h +0 -2
  59. package/ios/RnExecutorch/models/ocr/VerticalDetector.mm +2 -1
  60. package/ios/RnExecutorch/models/ocr/utils/OCRUtils.mm +0 -1
  61. package/ios/RnExecutorch/models/stt/Moonshine.mm +1 -6
  62. package/ios/RnExecutorch/models/stt/SpeechToTextBaseModel.mm +7 -11
  63. package/ios/RnExecutorch/models/stt/Whisper.mm +0 -5
  64. package/ios/RnExecutorch/models/{StyleTransferModel.h → style_transfer/StyleTransferModel.h} +1 -1
  65. package/ios/RnExecutorch/models/{StyleTransferModel.mm → style_transfer/StyleTransferModel.mm} +2 -3
  66. package/ios/RnExecutorch/models/text_embeddings/TextEmbeddingsModel.h +15 -0
  67. package/ios/RnExecutorch/models/text_embeddings/TextEmbeddingsModel.mm +45 -0
  68. package/ios/RnExecutorch/models/text_embeddings/TextEmbeddingsUtils.h +8 -0
  69. package/ios/RnExecutorch/models/text_embeddings/TextEmbeddingsUtils.mm +49 -0
  70. package/ios/RnExecutorch/utils/Conversions.h +15 -0
  71. package/ios/RnExecutorch/utils/ImageProcessor.h +0 -1
  72. package/ios/RnExecutorch/{models/classification/Utils.h → utils/Numerical.h} +0 -2
  73. package/ios/RnExecutorch/{models/classification/Utils.mm → utils/Numerical.mm} +0 -2
  74. package/ios/RnExecutorch/utils/ObjectDetectionUtils.mm +0 -2
  75. package/ios/RnExecutorch/utils/SFFT.mm +1 -1
  76. package/ios/RnExecutorch/utils/ScalarType.h +0 -2
  77. package/lib/module/Error.js +16 -2
  78. package/lib/module/Error.js.map +1 -1
  79. package/lib/module/constants/{llamaDefaults.js → llmDefaults.js} +7 -3
  80. package/lib/module/constants/llmDefaults.js.map +1 -0
  81. package/lib/module/constants/modelUrls.js +88 -27
  82. package/lib/module/constants/modelUrls.js.map +1 -1
  83. package/lib/module/constants/ocr/models.js +290 -0
  84. package/lib/module/constants/ocr/models.js.map +1 -0
  85. package/lib/module/constants/ocr/symbols.js +137 -2
  86. package/lib/module/constants/ocr/symbols.js.map +1 -1
  87. package/lib/module/constants/sttDefaults.js +50 -25
  88. package/lib/module/constants/sttDefaults.js.map +1 -1
  89. package/lib/module/controllers/LLMController.js +205 -0
  90. package/lib/module/controllers/LLMController.js.map +1 -0
  91. package/lib/module/controllers/OCRController.js +5 -10
  92. package/lib/module/controllers/OCRController.js.map +1 -1
  93. package/lib/module/controllers/SpeechToTextController.js +225 -122
  94. package/lib/module/controllers/SpeechToTextController.js.map +1 -1
  95. package/lib/module/controllers/VerticalOCRController.js +6 -10
  96. package/lib/module/controllers/VerticalOCRController.js.map +1 -1
  97. package/lib/module/hooks/computer_vision/useClassification.js +8 -23
  98. package/lib/module/hooks/computer_vision/useClassification.js.map +1 -1
  99. package/lib/module/hooks/computer_vision/useImageSegmentation.js +13 -0
  100. package/lib/module/hooks/computer_vision/useImageSegmentation.js.map +1 -0
  101. package/lib/module/hooks/computer_vision/useOCR.js +11 -6
  102. package/lib/module/hooks/computer_vision/useOCR.js.map +1 -1
  103. package/lib/module/hooks/computer_vision/useObjectDetection.js +8 -23
  104. package/lib/module/hooks/computer_vision/useObjectDetection.js.map +1 -1
  105. package/lib/module/hooks/computer_vision/useStyleTransfer.js +8 -23
  106. package/lib/module/hooks/computer_vision/useStyleTransfer.js.map +1 -1
  107. package/lib/module/hooks/computer_vision/useVerticalOCR.js +10 -7
  108. package/lib/module/hooks/computer_vision/useVerticalOCR.js.map +1 -1
  109. package/lib/module/hooks/general/useExecutorchModule.js +8 -36
  110. package/lib/module/hooks/general/useExecutorchModule.js.map +1 -1
  111. package/lib/module/hooks/natural_language_processing/useLLM.js +54 -63
  112. package/lib/module/hooks/natural_language_processing/useLLM.js.map +1 -1
  113. package/lib/module/hooks/natural_language_processing/useSpeechToText.js +15 -11
  114. package/lib/module/hooks/natural_language_processing/useSpeechToText.js.map +1 -1
  115. package/lib/module/hooks/natural_language_processing/useTextEmbeddings.js +14 -0
  116. package/lib/module/hooks/natural_language_processing/useTextEmbeddings.js.map +1 -0
  117. package/lib/module/hooks/natural_language_processing/useTokenizer.js +54 -0
  118. package/lib/module/hooks/natural_language_processing/useTokenizer.js.map +1 -0
  119. package/lib/module/hooks/useModule.js +18 -62
  120. package/lib/module/hooks/useModule.js.map +1 -1
  121. package/lib/module/index.js +16 -2
  122. package/lib/module/index.js.map +1 -1
  123. package/lib/module/modules/BaseModule.js +9 -10
  124. package/lib/module/modules/BaseModule.js.map +1 -1
  125. package/lib/module/modules/computer_vision/ClassificationModule.js +8 -5
  126. package/lib/module/modules/computer_vision/ClassificationModule.js.map +1 -1
  127. package/lib/module/modules/computer_vision/ImageSegmentationModule.js +28 -0
  128. package/lib/module/modules/computer_vision/ImageSegmentationModule.js.map +1 -0
  129. package/lib/module/modules/computer_vision/ObjectDetectionModule.js +8 -5
  130. package/lib/module/modules/computer_vision/ObjectDetectionModule.js.map +1 -1
  131. package/lib/module/modules/computer_vision/StyleTransferModule.js +8 -5
  132. package/lib/module/modules/computer_vision/StyleTransferModule.js.map +1 -1
  133. package/lib/module/modules/general/ExecutorchModule.js +8 -5
  134. package/lib/module/modules/general/ExecutorchModule.js.map +1 -1
  135. package/lib/module/modules/natural_language_processing/LLMModule.js +46 -27
  136. package/lib/module/modules/natural_language_processing/LLMModule.js.map +1 -1
  137. package/lib/module/modules/natural_language_processing/SpeechToTextModule.js +8 -5
  138. package/lib/module/modules/natural_language_processing/SpeechToTextModule.js.map +1 -1
  139. package/lib/module/modules/natural_language_processing/TextEmbeddingsModule.js +14 -0
  140. package/lib/module/modules/natural_language_processing/TextEmbeddingsModule.js.map +1 -0
  141. package/lib/module/modules/natural_language_processing/TokenizerModule.js +26 -0
  142. package/lib/module/modules/natural_language_processing/TokenizerModule.js.map +1 -0
  143. package/lib/module/native/NativeClassification.js.map +1 -1
  144. package/lib/module/native/NativeImageSegmentation.js +5 -0
  145. package/lib/module/native/NativeImageSegmentation.js.map +1 -0
  146. package/lib/module/native/NativeLLM.js.map +1 -1
  147. package/lib/module/native/NativeTextEmbeddings.js +5 -0
  148. package/lib/module/native/NativeTextEmbeddings.js.map +1 -0
  149. package/lib/module/native/NativeTokenizer.js +5 -0
  150. package/lib/module/native/NativeTokenizer.js.map +1 -0
  151. package/lib/module/native/RnExecutorchModules.js +18 -113
  152. package/lib/module/native/RnExecutorchModules.js.map +1 -1
  153. package/lib/module/types/common.js.map +1 -1
  154. package/lib/module/types/imageSegmentation.js +29 -0
  155. package/lib/module/types/imageSegmentation.js.map +1 -0
  156. package/lib/module/types/llm.js +7 -0
  157. package/lib/module/types/llm.js.map +1 -0
  158. package/lib/module/types/{object_detection.js → objectDetection.js} +1 -1
  159. package/lib/module/types/objectDetection.js.map +1 -0
  160. package/lib/module/types/ocr.js +2 -0
  161. package/lib/module/types/stt.js +82 -0
  162. package/lib/module/types/stt.js.map +1 -0
  163. package/lib/module/utils/ResourceFetcher.js +156 -0
  164. package/lib/module/utils/ResourceFetcher.js.map +1 -0
  165. package/lib/module/utils/llm.js +25 -0
  166. package/lib/module/utils/llm.js.map +1 -0
  167. package/lib/module/utils/stt.js +22 -0
  168. package/lib/module/utils/stt.js.map +1 -0
  169. package/lib/typescript/Error.d.ts +4 -1
  170. package/lib/typescript/Error.d.ts.map +1 -1
  171. package/lib/typescript/constants/{llamaDefaults.d.ts → llmDefaults.d.ts} +5 -5
  172. package/lib/typescript/constants/llmDefaults.d.ts.map +1 -0
  173. package/lib/typescript/constants/modelUrls.d.ts +74 -28
  174. package/lib/typescript/constants/modelUrls.d.ts.map +1 -1
  175. package/lib/typescript/constants/ocr/models.d.ts +285 -0
  176. package/lib/typescript/constants/ocr/models.d.ts.map +1 -0
  177. package/lib/typescript/constants/ocr/symbols.d.ts +73 -1
  178. package/lib/typescript/constants/ocr/symbols.d.ts.map +1 -1
  179. package/lib/typescript/constants/sttDefaults.d.ts +8 -13
  180. package/lib/typescript/constants/sttDefaults.d.ts.map +1 -1
  181. package/lib/typescript/controllers/LLMController.d.ts +46 -0
  182. package/lib/typescript/controllers/LLMController.d.ts.map +1 -0
  183. package/lib/typescript/controllers/OCRController.d.ts.map +1 -1
  184. package/lib/typescript/controllers/SpeechToTextController.d.ts +30 -16
  185. package/lib/typescript/controllers/SpeechToTextController.d.ts.map +1 -1
  186. package/lib/typescript/controllers/VerticalOCRController.d.ts +1 -1
  187. package/lib/typescript/controllers/VerticalOCRController.d.ts.map +1 -1
  188. package/lib/typescript/hooks/computer_vision/useClassification.d.ts +5 -5
  189. package/lib/typescript/hooks/computer_vision/useClassification.d.ts.map +1 -1
  190. package/lib/typescript/hooks/computer_vision/useImageSegmentation.d.ts +37 -0
  191. package/lib/typescript/hooks/computer_vision/useImageSegmentation.d.ts.map +1 -0
  192. package/lib/typescript/hooks/computer_vision/useOCR.d.ts +2 -1
  193. package/lib/typescript/hooks/computer_vision/useOCR.d.ts.map +1 -1
  194. package/lib/typescript/hooks/computer_vision/useObjectDetection.d.ts +5 -4
  195. package/lib/typescript/hooks/computer_vision/useObjectDetection.d.ts.map +1 -1
  196. package/lib/typescript/hooks/computer_vision/useStyleTransfer.d.ts +4 -2
  197. package/lib/typescript/hooks/computer_vision/useStyleTransfer.d.ts.map +1 -1
  198. package/lib/typescript/hooks/computer_vision/useVerticalOCR.d.ts +2 -1
  199. package/lib/typescript/hooks/computer_vision/useVerticalOCR.d.ts.map +1 -1
  200. package/lib/typescript/hooks/general/useExecutorchModule.d.ts +5 -6
  201. package/lib/typescript/hooks/general/useExecutorchModule.d.ts.map +1 -1
  202. package/lib/typescript/hooks/natural_language_processing/useLLM.d.ts +6 -6
  203. package/lib/typescript/hooks/natural_language_processing/useLLM.d.ts.map +1 -1
  204. package/lib/typescript/hooks/natural_language_processing/useSpeechToText.d.ts +7 -3
  205. package/lib/typescript/hooks/natural_language_processing/useSpeechToText.d.ts.map +1 -1
  206. package/lib/typescript/hooks/natural_language_processing/useTextEmbeddings.d.ts +13 -0
  207. package/lib/typescript/hooks/natural_language_processing/useTextEmbeddings.d.ts.map +1 -0
  208. package/lib/typescript/hooks/natural_language_processing/useTokenizer.d.ts +16 -0
  209. package/lib/typescript/hooks/natural_language_processing/useTokenizer.d.ts.map +1 -0
  210. package/lib/typescript/hooks/useModule.d.ts +11 -10
  211. package/lib/typescript/hooks/useModule.d.ts.map +1 -1
  212. package/lib/typescript/index.d.ts +15 -2
  213. package/lib/typescript/index.d.ts.map +1 -1
  214. package/lib/typescript/modules/BaseModule.d.ts +4 -5
  215. package/lib/typescript/modules/BaseModule.d.ts.map +1 -1
  216. package/lib/typescript/modules/computer_vision/ClassificationModule.d.ts +7 -7
  217. package/lib/typescript/modules/computer_vision/ClassificationModule.d.ts.map +1 -1
  218. package/lib/typescript/modules/computer_vision/ImageSegmentationModule.d.ts +32 -0
  219. package/lib/typescript/modules/computer_vision/ImageSegmentationModule.d.ts.map +1 -0
  220. package/lib/typescript/modules/computer_vision/ObjectDetectionModule.d.ts +6 -5
  221. package/lib/typescript/modules/computer_vision/ObjectDetectionModule.d.ts.map +1 -1
  222. package/lib/typescript/modules/computer_vision/StyleTransferModule.d.ts +6 -5
  223. package/lib/typescript/modules/computer_vision/StyleTransferModule.d.ts.map +1 -1
  224. package/lib/typescript/modules/general/ExecutorchModule.d.ts +4 -3
  225. package/lib/typescript/modules/general/ExecutorchModule.d.ts.map +1 -1
  226. package/lib/typescript/modules/natural_language_processing/LLMModule.d.ts +19 -5
  227. package/lib/typescript/modules/natural_language_processing/LLMModule.d.ts.map +1 -1
  228. package/lib/typescript/modules/natural_language_processing/SpeechToTextModule.d.ts +7 -4
  229. package/lib/typescript/modules/natural_language_processing/SpeechToTextModule.d.ts.map +1 -1
  230. package/lib/typescript/modules/natural_language_processing/TextEmbeddingsModule.d.ts +8 -0
  231. package/lib/typescript/modules/natural_language_processing/TextEmbeddingsModule.d.ts.map +1 -0
  232. package/lib/typescript/modules/natural_language_processing/TokenizerModule.d.ts +12 -0
  233. package/lib/typescript/modules/natural_language_processing/TokenizerModule.d.ts.map +1 -0
  234. package/lib/typescript/native/NativeClassification.d.ts.map +1 -1
  235. package/lib/typescript/native/NativeImageSegmentation.d.ts +10 -0
  236. package/lib/typescript/native/NativeImageSegmentation.d.ts.map +1 -0
  237. package/lib/typescript/native/NativeLLM.d.ts +3 -4
  238. package/lib/typescript/native/NativeLLM.d.ts.map +1 -1
  239. package/lib/typescript/native/NativeObjectDetection.d.ts +1 -1
  240. package/lib/typescript/native/NativeObjectDetection.d.ts.map +1 -1
  241. package/lib/typescript/native/NativeSpeechToText.d.ts +2 -2
  242. package/lib/typescript/native/NativeSpeechToText.d.ts.map +1 -1
  243. package/lib/typescript/native/NativeTextEmbeddings.d.ts +8 -0
  244. package/lib/typescript/native/NativeTextEmbeddings.d.ts.map +1 -0
  245. package/lib/typescript/native/NativeTokenizer.d.ts +12 -0
  246. package/lib/typescript/native/NativeTokenizer.d.ts.map +1 -0
  247. package/lib/typescript/native/RnExecutorchModules.d.ts +18 -41
  248. package/lib/typescript/native/RnExecutorchModules.d.ts.map +1 -1
  249. package/lib/typescript/types/common.d.ts +1 -26
  250. package/lib/typescript/types/common.d.ts.map +1 -1
  251. package/lib/typescript/types/imageSegmentation.d.ts +25 -0
  252. package/lib/typescript/types/imageSegmentation.d.ts.map +1 -0
  253. package/lib/typescript/types/llm.d.ts +38 -0
  254. package/lib/typescript/types/llm.d.ts.map +1 -0
  255. package/lib/typescript/types/{object_detection.d.ts → objectDetection.d.ts} +1 -1
  256. package/lib/typescript/types/objectDetection.d.ts.map +1 -0
  257. package/lib/typescript/types/ocr.d.ts +2 -1
  258. package/lib/typescript/types/ocr.d.ts.map +1 -1
  259. package/lib/typescript/types/stt.d.ts +91 -0
  260. package/lib/typescript/types/stt.d.ts.map +1 -0
  261. package/lib/typescript/utils/ResourceFetcher.d.ts +17 -0
  262. package/lib/typescript/utils/ResourceFetcher.d.ts.map +1 -0
  263. package/lib/typescript/utils/llm.d.ts +3 -0
  264. package/lib/typescript/utils/llm.d.ts.map +1 -0
  265. package/lib/typescript/utils/stt.d.ts +2 -0
  266. package/lib/typescript/utils/stt.d.ts.map +1 -0
  267. package/package.json +13 -49
  268. package/react-native-executorch.podspec +1 -1
  269. package/src/Error.ts +16 -3
  270. package/src/constants/llmDefaults.ts +14 -0
  271. package/src/constants/modelUrls.ts +146 -39
  272. package/src/constants/ocr/models.ts +453 -0
  273. package/src/constants/ocr/symbols.ts +147 -3
  274. package/src/constants/sttDefaults.ts +55 -37
  275. package/src/controllers/LLMController.ts +286 -0
  276. package/src/controllers/OCRController.ts +14 -28
  277. package/src/controllers/SpeechToTextController.ts +318 -180
  278. package/src/controllers/VerticalOCRController.ts +17 -32
  279. package/src/hooks/computer_vision/useClassification.ts +11 -26
  280. package/src/hooks/computer_vision/useImageSegmentation.ts +18 -0
  281. package/src/hooks/computer_vision/useOCR.ts +17 -5
  282. package/src/hooks/computer_vision/useObjectDetection.ts +10 -24
  283. package/src/hooks/computer_vision/useStyleTransfer.ts +9 -25
  284. package/src/hooks/computer_vision/useVerticalOCR.ts +11 -4
  285. package/src/hooks/general/useExecutorchModule.ts +10 -50
  286. package/src/hooks/natural_language_processing/useLLM.ts +80 -97
  287. package/src/hooks/natural_language_processing/useSpeechToText.ts +39 -12
  288. package/src/hooks/natural_language_processing/useTextEmbeddings.ts +18 -0
  289. package/src/hooks/natural_language_processing/useTokenizer.ts +61 -0
  290. package/src/hooks/useModule.ts +32 -92
  291. package/src/index.tsx +16 -2
  292. package/src/modules/BaseModule.ts +16 -26
  293. package/src/modules/computer_vision/ClassificationModule.ts +13 -8
  294. package/src/modules/computer_vision/ImageSegmentationModule.ts +39 -0
  295. package/src/modules/computer_vision/ObjectDetectionModule.ts +13 -8
  296. package/src/modules/computer_vision/StyleTransferModule.ts +13 -8
  297. package/src/modules/general/ExecutorchModule.ts +11 -6
  298. package/src/modules/natural_language_processing/LLMModule.ts +64 -51
  299. package/src/modules/natural_language_processing/SpeechToTextModule.ts +25 -10
  300. package/src/modules/natural_language_processing/TextEmbeddingsModule.ts +18 -0
  301. package/src/modules/natural_language_processing/TokenizerModule.ts +34 -0
  302. package/src/native/NativeClassification.ts +0 -1
  303. package/src/native/NativeImageSegmentation.ts +14 -0
  304. package/src/native/NativeLLM.ts +3 -10
  305. package/src/native/NativeObjectDetection.ts +1 -1
  306. package/src/native/NativeSpeechToText.ts +2 -2
  307. package/src/native/NativeTextEmbeddings.ts +9 -0
  308. package/src/native/NativeTokenizer.ts +13 -0
  309. package/src/native/RnExecutorchModules.ts +54 -234
  310. package/src/types/common.ts +1 -44
  311. package/src/types/imageSegmentation.ts +25 -0
  312. package/src/types/llm.ts +57 -0
  313. package/src/types/ocr.ts +3 -1
  314. package/src/types/stt.ts +93 -0
  315. package/src/utils/ResourceFetcher.ts +196 -0
  316. package/src/utils/llm.ts +34 -0
  317. package/src/utils/stt.ts +28 -0
  318. package/android/src/main/java/com/swmansion/rnexecutorch/utils/llms/ConversationManager.kt +0 -68
  319. package/ios/ExecutorchLib.xcframework/ios-arm64-simulator/ExecutorchLib.framework/_CodeSignature/CodeResources +0 -124
  320. package/ios/RnExecutorch/utils/llms/Constants.h +0 -6
  321. package/ios/RnExecutorch/utils/llms/Constants.mm +0 -23
  322. package/ios/RnExecutorch/utils/llms/ConversationManager.h +0 -26
  323. package/ios/RnExecutorch/utils/llms/ConversationManager.mm +0 -71
  324. package/lib/module/constants/llamaDefaults.js.map +0 -1
  325. package/lib/module/modules/computer_vision/BaseCVModule.js +0 -14
  326. package/lib/module/modules/computer_vision/BaseCVModule.js.map +0 -1
  327. package/lib/module/types/object_detection.js.map +0 -1
  328. package/lib/module/utils/fetchResource.js +0 -93
  329. package/lib/module/utils/fetchResource.js.map +0 -1
  330. package/lib/module/utils/listDownloadedResources.js +0 -13
  331. package/lib/module/utils/listDownloadedResources.js.map +0 -1
  332. package/lib/typescript/constants/llamaDefaults.d.ts.map +0 -1
  333. package/lib/typescript/modules/computer_vision/BaseCVModule.d.ts +0 -9
  334. package/lib/typescript/modules/computer_vision/BaseCVModule.d.ts.map +0 -1
  335. package/lib/typescript/types/object_detection.d.ts.map +0 -1
  336. package/lib/typescript/utils/fetchResource.d.ts +0 -3
  337. package/lib/typescript/utils/fetchResource.d.ts.map +0 -1
  338. package/lib/typescript/utils/listDownloadedResources.d.ts +0 -3
  339. package/lib/typescript/utils/listDownloadedResources.d.ts.map +0 -1
  340. package/src/constants/llamaDefaults.ts +0 -9
  341. package/src/modules/computer_vision/BaseCVModule.ts +0 -22
  342. package/src/utils/fetchResource.ts +0 -106
  343. package/src/utils/listDownloadedResources.ts +0 -12
  344. /package/src/types/{object_detection.ts → objectDetection.ts} +0 -0
@@ -0,0 +1,146 @@
1
+ #import "ImageSegmentationModel.h"
2
+ #import "../../utils/Conversions.h"
3
+ #import "../../utils/ImageProcessor.h"
4
+ #import "../../utils/Numerical.h"
5
+ #import "Constants.h"
6
+ #import <unordered_set>
7
+
8
+ @interface ImageSegmentationModel ()
9
+ - (NSArray *)preprocess:(cv::Mat &)input;
10
+ - (NSDictionary *)postprocess:(NSArray *)output
11
+ returnClasses:(NSArray *)classesOfInterest
12
+ resize:(BOOL)resize;
13
+ @end
14
+
15
+ @implementation ImageSegmentationModel {
16
+ cv::Size originalSize;
17
+ }
18
+
19
+ - (cv::Size)getModelImageSize {
20
+ NSArray *inputShape = [module getInputShape:@0];
21
+ NSNumber *widthNumber = inputShape.lastObject;
22
+ NSNumber *heightNumber = inputShape[inputShape.count - 2];
23
+
24
+ int height = [heightNumber intValue];
25
+ int width = [widthNumber intValue];
26
+
27
+ return cv::Size(height, width);
28
+ }
29
+
30
+ - (NSArray *)preprocess:(cv::Mat &)input {
31
+ originalSize = cv::Size(input.cols, input.rows);
32
+
33
+ cv::Size modelImageSize = [self getModelImageSize];
34
+ cv::Mat output;
35
+ cv::resize(input, output, modelImageSize);
36
+
37
+ NSArray *modelInput = [ImageProcessor matToNSArray:output];
38
+ return modelInput;
39
+ }
40
+
41
+ std::vector<cv::Mat> extractResults(NSArray *result, std::size_t numLabels,
42
+ cv::Size modelImageSize,
43
+ cv::Size originalSize, BOOL resize) {
44
+ std::size_t numModelPixels = modelImageSize.height * modelImageSize.width;
45
+
46
+ std::vector<cv::Mat> resizedLabelScores(numLabels);
47
+ for (std::size_t label = 0; label < numLabels; ++label) {
48
+ cv::Mat labelMat = cv::Mat(modelImageSize, CV_64F);
49
+
50
+ for (std::size_t pixel = 0; pixel < numModelPixels; ++pixel) {
51
+ int row = pixel / modelImageSize.width;
52
+ int col = pixel % modelImageSize.width;
53
+ labelMat.at<double>(row, col) =
54
+ [result[label * numModelPixels + pixel] doubleValue];
55
+ }
56
+
57
+ if (resize) {
58
+ cv::resize(labelMat, resizedLabelScores[label], originalSize);
59
+ } else {
60
+ resizedLabelScores[label] = std::move(labelMat);
61
+ }
62
+ }
63
+ return resizedLabelScores;
64
+ }
65
+
66
+ void adjustScoresPerPixel(std::vector<cv::Mat> &labelScores, cv::Mat &argMax,
67
+ cv::Size outputSize, std::size_t numLabels) {
68
+ std::size_t numOutputPixels = outputSize.height * outputSize.width;
69
+ for (std::size_t pixel = 0; pixel < numOutputPixels; ++pixel) {
70
+ int row = pixel / outputSize.width;
71
+ int col = pixel % outputSize.width;
72
+ std::vector<double> scores;
73
+ scores.reserve(numLabels);
74
+ for (const auto &mat : labelScores) {
75
+ scores.push_back(mat.at<double>(row, col));
76
+ }
77
+
78
+ std::vector<double> adjustedScores = softmax(scores);
79
+
80
+ for (std::size_t label = 0; label < numLabels; ++label) {
81
+ labelScores[label].at<double>(row, col) = adjustedScores[label];
82
+ }
83
+
84
+ auto maxIt = std::max_element(scores.begin(), scores.end());
85
+ argMax.at<int>(row, col) = std::distance(scores.begin(), maxIt);
86
+ }
87
+ }
88
+
89
+ - (NSDictionary *)postprocess:(NSArray *)output
90
+ returnClasses:(NSArray *)classesOfInterest
91
+ resize:(BOOL)resize {
92
+ cv::Size modelImageSize = [self getModelImageSize];
93
+
94
+ std::size_t numLabels = deeplabv3_resnet50_labels.size();
95
+
96
+ NSAssert((std::size_t)output.count ==
97
+ numLabels * modelImageSize.height * modelImageSize.width,
98
+ @"Model generated unexpected output size.");
99
+
100
+ // For each label extract it's matrix,
101
+ // and rescale it to the original size if `resize`
102
+ std::vector<cv::Mat> resizedLabelScores =
103
+ extractResults(output, numLabels, modelImageSize, originalSize, resize);
104
+
105
+ cv::Size outputSize = resize ? originalSize : modelImageSize;
106
+ cv::Mat argMax = cv::Mat(outputSize, CV_32S);
107
+
108
+ // For each pixel apply softmax across all the labels and calculate the argMax
109
+ adjustScoresPerPixel(resizedLabelScores, argMax, outputSize, numLabels);
110
+
111
+ std::unordered_set<std::string> labelSet;
112
+
113
+ for (id label in classesOfInterest) {
114
+ labelSet.insert(std::string([label UTF8String]));
115
+ }
116
+
117
+ NSMutableDictionary *result = [NSMutableDictionary dictionary];
118
+
119
+ // Convert to NSArray and populate the final dictionary
120
+ for (std::size_t label = 0; label < numLabels; ++label) {
121
+ if (labelSet.contains(deeplabv3_resnet50_labels[label])) {
122
+ NSString *labelString = @(deeplabv3_resnet50_labels[label].c_str());
123
+ NSArray *arr = simpleMatToNSArray<double>(resizedLabelScores[label]);
124
+ result[labelString] = arr;
125
+ }
126
+ }
127
+
128
+ result[@"ARGMAX"] = simpleMatToNSArray<int>(argMax);
129
+
130
+ return result;
131
+ }
132
+
133
+ - (NSDictionary *)runModel:(cv::Mat &)input
134
+ returnClasses:(NSArray *)classesOfInterest
135
+ resize:(BOOL)resize {
136
+ NSArray *modelInput = [self preprocess:input];
137
+ NSArray *result = [self forward:@[ modelInput ]];
138
+
139
+ NSDictionary *output = [self postprocess:result[0]
140
+ returnClasses:classesOfInterest
141
+ resize:resize];
142
+
143
+ return output;
144
+ }
145
+
146
+ @end
@@ -1,7 +1,6 @@
1
1
  #include "SSDLiteLargeModel.hpp"
2
2
  #include "../../utils/ObjectDetectionUtils.hpp"
3
3
  #include "ImageProcessor.h"
4
- #include <vector>
5
4
 
6
5
  float constexpr iouThreshold = 0.55;
7
6
  float constexpr detectionThreshold = 0.7;
@@ -54,7 +53,7 @@ int constexpr modelInputHeight = 320;
54
53
  int inputImageWidth = size.width;
55
54
  int inputImageHeight = size.height;
56
55
  NSArray *modelInput = [self preprocess:input];
57
- NSArray *forwardResult = [self forward:modelInput];
56
+ NSArray *forwardResult = [self forward:@[ modelInput ]];
58
57
  NSArray *output =
59
58
  [self postprocess:forwardResult
60
59
  widthRatio:inputImageWidth / (float)modelInputWidth
@@ -1,7 +1,5 @@
1
1
  #import "BaseModel.h"
2
- #import "RecognitionHandler.h"
3
2
  #import "opencv2/opencv.hpp"
4
- #import "utils/Constants.h"
5
3
 
6
4
  @interface Detector : BaseModel
7
5
 
@@ -1,5 +1,6 @@
1
1
  #import "Detector.h"
2
2
  #import "../../utils/ImageProcessor.h"
3
+ #import "utils/Constants.h"
3
4
  #import "utils/DetectorUtils.h"
4
5
  #import "utils/OCRUtils.h"
5
6
 
@@ -92,7 +93,7 @@
92
93
 
93
94
  - (NSArray *)runModel:(cv::Mat &)input {
94
95
  NSArray *modelInput = [self preprocess:input];
95
- NSArray *modelResult = [self forward:modelInput];
96
+ NSArray *modelResult = [self forward:@[ modelInput ]];
96
97
  NSArray *result = [self postprocess:modelResult];
97
98
  return result;
98
99
  }
@@ -3,10 +3,11 @@
3
3
  @interface RecognitionHandler : NSObject
4
4
 
5
5
  - (instancetype)initWithSymbols:(NSString *)symbols;
6
- - (void)loadRecognizers:(NSString *)largeRecognizerPath
7
- mediumRecognizerPath:(NSString *)mediumRecognizerPath
8
- smallRecognizerPath:(NSString *)smallRecognizerPath
9
- completion:(void (^)(BOOL, NSNumber *))completion;
6
+
7
+ - (NSNumber *)loadRecognizers:(NSString *)largeRecognizerPath
8
+ mediumRecognizerPath:(NSString *)mediumRecognizerPath
9
+ smallRecognizerPath:(NSString *)smallRecognizerPath;
10
+
10
11
  - (NSArray *)recognize:(NSArray<NSDictionary *> *)bBoxesList
11
12
  imgGray:(cv::Mat)imgGray
12
13
  desiredWidth:(int)desiredWidth
@@ -1,12 +1,9 @@
1
1
  #import "RecognitionHandler.h"
2
- #import "../../utils/ImageProcessor.h"
3
2
  #import "./utils/CTCLabelConverter.h"
4
3
  #import "./utils/Constants.h"
5
4
  #import "./utils/OCRUtils.h"
6
5
  #import "./utils/RecognizerUtils.h"
7
- #import "ExecutorchLib/ETModel.h"
8
6
  #import "Recognizer.h"
9
- #import <React/RCTBridgeModule.h>
10
7
 
11
8
  /*
12
9
  RecognitionHandler class is responsible for loading and choosing the
@@ -34,15 +31,12 @@
34
31
  return self;
35
32
  }
36
33
 
37
- - (void)loadRecognizers:(NSString *)largeRecognizerPath
38
- mediumRecognizerPath:(NSString *)mediumRecognizerPath
39
- smallRecognizerPath:(NSString *)smallRecognizerPath
40
- completion:(void (^)(BOOL, NSNumber *))completion {
41
- dispatch_group_t group = dispatch_group_create();
42
- __block BOOL allSuccessful = YES;
43
-
34
+ - (NSNumber *)loadRecognizers:(NSString *)largeRecognizerPath
35
+ mediumRecognizerPath:(NSString *)mediumRecognizerPath
36
+ smallRecognizerPath:(NSString *)smallRecognizerPath {
44
37
  NSArray<Recognizer *> *recognizers =
45
38
  @[ recognizerLarge, recognizerMedium, recognizerSmall ];
39
+
46
40
  NSArray<NSString *> *paths =
47
41
  @[ largeRecognizerPath, mediumRecognizerPath, smallRecognizerPath ];
48
42
 
@@ -50,24 +44,13 @@
50
44
  Recognizer *recognizer = recognizers[i];
51
45
  NSString *path = paths[i];
52
46
 
53
- dispatch_group_enter(group);
54
- [recognizer loadModel:[NSURL URLWithString:path]
55
- completion:^(BOOL success, NSNumber *errorCode) {
56
- if (!success) {
57
- allSuccessful = NO;
58
- dispatch_group_leave(group);
59
- completion(NO, errorCode);
60
- return;
61
- }
62
- dispatch_group_leave(group);
63
- }];
47
+ NSNumber *errorCode = [recognizer loadModel:path];
48
+ if ([errorCode intValue] != 0) {
49
+ return errorCode;
50
+ }
64
51
  }
65
52
 
66
- dispatch_group_notify(group, dispatch_get_main_queue(), ^{
67
- if (allSuccessful) {
68
- completion(YES, @(0));
69
- }
70
- });
53
+ return @0;
71
54
  }
72
55
 
73
56
  - (NSArray *)runModel:(cv::Mat)croppedImage {
@@ -1,7 +1,6 @@
1
1
  #import "Recognizer.h"
2
2
  #import "../../utils/ImageProcessor.h"
3
3
  #import "RecognizerUtils.h"
4
- #import "utils/OCRUtils.h"
5
4
 
6
5
  /*
7
6
  The model used as detector is based on CRNN paper.
@@ -69,7 +68,7 @@
69
68
 
70
69
  - (NSArray *)runModel:(cv::Mat &)input {
71
70
  NSArray *modelInput = [self preprocess:input];
72
- NSArray *modelResult = [self forward:modelInput];
71
+ NSArray *modelResult = [self forward:@[ modelInput ]];
73
72
  NSArray *result = [self postprocess:modelResult];
74
73
 
75
74
  return result;
@@ -1,7 +1,5 @@
1
1
  #import "BaseModel.h"
2
- #import "RecognitionHandler.h"
3
2
  #import "opencv2/opencv.hpp"
4
- #import "utils/Constants.h"
5
3
 
6
4
  @interface VerticalDetector : BaseModel
7
5
 
@@ -1,5 +1,6 @@
1
1
  #import "VerticalDetector.h"
2
2
  #import "../../utils/ImageProcessor.h"
3
+ #import "utils/Constants.h"
3
4
  #import "utils/DetectorUtils.h"
4
5
  #import "utils/OCRUtils.h"
5
6
 
@@ -109,7 +110,7 @@
109
110
 
110
111
  - (NSArray *)runModel:(cv::Mat &)input {
111
112
  NSArray *modelInput = [self preprocess:input];
112
- NSArray *modelResult = [self forward:modelInput];
113
+ NSArray *modelResult = [self forward:@[ modelInput ]];
113
114
  NSArray *result = [self postprocess:modelResult];
114
115
  return result;
115
116
  }
@@ -1,5 +1,4 @@
1
1
  #import "OCRUtils.h"
2
- #import "RecognizerUtils.h"
3
2
 
4
3
  @implementation OCRUtils
5
4
 
@@ -1,9 +1,4 @@
1
1
  #import "Moonshine.hpp"
2
- #import "ExecutorchLib/ETModel.h"
3
- #import "MoonshineEncoder.hpp"
4
- #import "Moonshinedecoder.hpp"
5
- #import <Foundation/Foundation.h>
6
- #import <UIKit/UIKit.h>
7
2
 
8
3
  @implementation Moonshine {
9
4
  MoonshineEncoder *encoder;
@@ -66,4 +61,4 @@
66
61
  }];
67
62
  }
68
63
 
69
- @end
64
+ @end
@@ -1,6 +1,4 @@
1
1
  #import "SpeechToTextBaseModel.hpp"
2
- #import <Foundation/Foundation.h>
3
- #import <UIKit/UIKit.h>
4
2
 
5
3
  @implementation SpeechToTextBaseModel
6
4
 
@@ -8,16 +6,14 @@
8
6
  withSource:(NSString *)source
9
7
  onSuccess:(void (^)(void))success
10
8
  onFailure:(void (^)(NSString *))failure {
9
+ NSNumber *errorCode = [model loadModel:source];
11
10
 
12
- [model loadModel:[NSURL URLWithString:source]
13
- completion:^(BOOL isSuccess, NSNumber *errorCode) {
14
- if (isSuccess) {
15
- success();
16
- } else {
17
- failure([NSString
18
- stringWithFormat:@"%ld", (long)[errorCode longValue]]);
19
- }
20
- }];
11
+ if ([errorCode intValue] != 0) {
12
+ failure([NSString stringWithFormat:@"%ld", (long)[errorCode longValue]]);
13
+ return;
14
+ }
15
+
16
+ success();
21
17
  }
22
18
 
23
19
  @end
@@ -1,11 +1,6 @@
1
1
  #import "Whisper.hpp"
2
- #import "../../utils/SFFT.hpp"
3
- #import "../../utils/ScalarType.h"
4
- #import "ExecutorchLib/ETModel.h"
5
2
  #import "WhisperEncoder.hpp"
6
3
  #import "Whisperdecoder.hpp"
7
- #import <Foundation/Foundation.h>
8
- #import <UIKit/UIKit.h>
9
4
 
10
5
  @implementation Whisper {
11
6
  WhisperEncoder *encoder;
@@ -1,4 +1,4 @@
1
- #import "BaseModel.h"
1
+ #import "../BaseModel.h"
2
2
  #import "opencv2/opencv.hpp"
3
3
 
4
4
  @interface StyleTransferModel : BaseModel
@@ -1,6 +1,5 @@
1
1
  #import "StyleTransferModel.h"
2
- #import "../utils/ImageProcessor.h"
3
- #import "opencv2/opencv.hpp"
2
+ #import "../../utils/ImageProcessor.h"
4
3
 
5
4
  @implementation StyleTransferModel {
6
5
  cv::Size originalSize;
@@ -42,7 +41,7 @@
42
41
 
43
42
  - (cv::Mat)runModel:(cv::Mat &)input {
44
43
  NSArray *modelInput = [self preprocess:input];
45
- NSArray *result = [self forward:modelInput];
44
+ NSArray *result = [self forward:@[ modelInput ]];
46
45
  input = [self postprocess:result[0]];
47
46
 
48
47
  return input;
@@ -0,0 +1,15 @@
1
+ #import "BaseModel.h"
2
+ #import "ExecutorchLib/HuggingFaceTokenizer.h"
3
+
4
+ @interface TextEmbeddingsModel : BaseModel {
5
+ @protected
6
+ HuggingFaceTokenizer *tokenizer;
7
+ }
8
+
9
+ - (void)loadTokenizer:(NSString *)tokenizerSource;
10
+ - (NSArray *)preprocess:(NSString *)input;
11
+ - (NSArray *)runModel:(NSString *)input;
12
+ - (NSArray *)postprocess:(NSArray *)input
13
+ attentionMask:(NSArray *)attentionMask;
14
+
15
+ @end
@@ -0,0 +1,45 @@
1
+ #import "TextEmbeddingsModel.h"
2
+ #import "TextEmbeddingsUtils.h"
3
+
4
+ @implementation TextEmbeddingsModel
5
+
6
+ - (NSArray *)preprocess:(NSString *)input {
7
+ NSArray *input_ids = [self->tokenizer encode:input];
8
+ NSMutableArray *attention_mask = [NSMutableArray new];
9
+ for (int i = 0; i < [input_ids count]; i++) {
10
+ [attention_mask addObject:@((int)([input_ids[i] intValue] != 0))];
11
+ }
12
+ return @[ input_ids, attention_mask ]; // [2, tokens]
13
+ }
14
+
15
+ - (NSArray *)postprocess:(NSArray *)modelOutput // [tokens * embedding_dim]
16
+ attentionMask:(NSArray *)attentionMask // [tokens]
17
+ {
18
+ NSArray *embeddings = [TextEmbeddingsUtils meanPooling:modelOutput
19
+ attentionMask:attentionMask];
20
+ return [TextEmbeddingsUtils normalize:embeddings];
21
+ }
22
+
23
+ - (NSArray *)runModel:(NSString *)input {
24
+ NSArray *modelInput = [self preprocess:input];
25
+
26
+ NSMutableArray *inputTypes = [NSMutableArray arrayWithObjects:@4, @4, nil];
27
+ NSMutableArray *shapes = [NSMutableArray new];
28
+
29
+ NSNumber *tokenCount = @([modelInput[0] count]);
30
+ for (__unused id _ in modelInput) {
31
+ [shapes addObject:[NSMutableArray arrayWithObjects:@1, tokenCount, nil]];
32
+ }
33
+
34
+ NSArray *modelOutput = [self forward:modelInput
35
+ shapes:shapes
36
+ inputTypes:inputTypes];
37
+ return [self postprocess:modelOutput[0] attentionMask:modelInput[1]];
38
+ }
39
+
40
+ - (void)loadTokenizer:(NSString *)tokenizerSource {
41
+ tokenizer =
42
+ [[HuggingFaceTokenizer alloc] initWithTokenizerPath:tokenizerSource];
43
+ }
44
+
45
+ @end
@@ -0,0 +1,8 @@
1
+ @interface TextEmbeddingsUtils : NSObject
2
+
3
+ + (NSArray *)meanPooling:(NSArray *)modelOutput
4
+ attentionMask:(NSArray *)attentionMask;
5
+
6
+ + (NSArray *)normalize:(NSArray *)embeddings;
7
+
8
+ @end
@@ -0,0 +1,49 @@
1
+ #import "TextEmbeddingsUtils.h"
2
+
3
+ @implementation TextEmbeddingsUtils
4
+
5
+ + (NSArray *)meanPooling:(NSArray *)modelOutput
6
+ attentionMask:(NSArray *)attentionMask {
7
+ NSInteger modelOutputLength = [modelOutput count];
8
+ NSInteger attentionMaskLength = [attentionMask count];
9
+ NSInteger embeddingDim = modelOutputLength / attentionMaskLength;
10
+
11
+ NSMutableArray *result = [NSMutableArray arrayWithCapacity:embeddingDim];
12
+
13
+ double sumMask = 0.0;
14
+ for (NSNumber *maskValue in attentionMask) {
15
+ sumMask += [maskValue intValue];
16
+ }
17
+ sumMask = fmax(sumMask, 1e-9);
18
+
19
+ for (NSInteger i = 0; i < embeddingDim; i++) {
20
+ double sum = 0;
21
+ for (NSInteger j = 0; j < attentionMaskLength; j++) {
22
+ sum += [modelOutput[j * embeddingDim + i] doubleValue] *
23
+ [attentionMask[j] intValue];
24
+ }
25
+ [result addObject:@(sum / sumMask)];
26
+ }
27
+
28
+ return result;
29
+ }
30
+
31
+ + (NSArray *)normalize:(NSArray *)embeddings {
32
+ NSInteger embeddingDim = [embeddings count];
33
+ double sum = 0.0;
34
+
35
+ for (NSNumber *value in embeddings) {
36
+ sum += [value doubleValue] * [value doubleValue];
37
+ }
38
+ sum = fmax(sqrt(sum), 1e-9);
39
+
40
+ NSMutableArray *normalizedResult =
41
+ [NSMutableArray arrayWithCapacity:embeddingDim];
42
+ for (NSNumber *value in embeddings) {
43
+ [normalizedResult addObject:@([value doubleValue] / sum)];
44
+ }
45
+
46
+ return normalizedResult;
47
+ }
48
+
49
+ @end
@@ -0,0 +1,15 @@
1
+ #import "opencv2/opencv.hpp"
2
+
3
+ // Convert a matrix containing a single value per cell to a NSArray
4
+ template <typename T>
5
+ NSArray* simpleMatToNSArray(const cv::Mat& mat) {
6
+ std::size_t numPixels = mat.rows * mat.cols;
7
+ NSMutableArray *arr = [[NSMutableArray alloc] initWithCapacity:numPixels];
8
+
9
+ for (std::size_t x = 0; x < mat.rows; ++x) {
10
+ for (std::size_t y = 0; y < mat.cols; ++y) {
11
+ arr[x * mat.cols + y] = @(mat.at<T>(x, y));
12
+ }
13
+ }
14
+ return arr;
15
+ }
@@ -1,4 +1,3 @@
1
- #import <UIKit/UIKit.h>
2
1
  #import <opencv2/opencv.hpp>
3
2
 
4
3
  @interface ImageProcessor : NSObject
@@ -1,3 +1 @@
1
- #include <vector>
2
-
3
1
  std::vector<double> softmax(const std::vector<double> &v);
@@ -1,6 +1,4 @@
1
1
  #include "Utils.h"
2
- #include <cmath>
3
- #include <vector>
4
2
 
5
3
  std::vector<double> softmax(const std::vector<double> &v) {
6
4
  std::vector<double> result(v.size());
@@ -1,7 +1,5 @@
1
1
  #include "ObjectDetectionUtils.hpp"
2
2
  #include "Constants.h"
3
- #include <map>
4
- #include <vector>
5
3
 
6
4
  NSString *floatLabelToNSString(float label) {
7
5
  int intLabel = static_cast<int>(label);
@@ -68,4 +68,4 @@
68
68
  free(a.imagp);
69
69
  }
70
70
 
71
- @end
71
+ @end
@@ -1,8 +1,6 @@
1
1
  #ifndef ScalarType_h
2
2
  #define ScalarType_h
3
3
 
4
- #import <Foundation/Foundation.h>
5
-
6
4
  @interface ScalarType : NSObject
7
5
 
8
6
  @property(class, nonatomic, readonly) NSNumber *Int8;
@@ -1,25 +1,36 @@
1
1
  "use strict";
2
2
 
3
3
  export let ETError = /*#__PURE__*/function (ETError) {
4
+ // React-native-ExecuTorch errors
4
5
  ETError[ETError["UndefinedError"] = 101] = "UndefinedError";
5
6
  ETError[ETError["ModuleNotLoaded"] = 102] = "ModuleNotLoaded";
6
7
  ETError[ETError["FileWriteFailed"] = 103] = "FileWriteFailed";
7
8
  ETError[ETError["ModelGenerating"] = 104] = "ModelGenerating";
8
9
  ETError[ETError["LanguageNotSupported"] = 105] = "LanguageNotSupported";
9
10
  ETError[ETError["InvalidModelSource"] = 255] = "InvalidModelSource";
11
+ //SpeechToText errors
12
+ ETError[ETError["MultilingualConfiguration"] = 160] = "MultilingualConfiguration";
13
+ ETError[ETError["MissingDataChunk"] = 161] = "MissingDataChunk";
14
+ ETError[ETError["StreamingNotStarted"] = 162] = "StreamingNotStarted";
15
+ // ExecuTorch mapped errors
16
+ // Based on: https://github.com/pytorch/executorch/blob/main/runtime/core/error.h
17
+ // System errors
10
18
  ETError[ETError["Ok"] = 0] = "Ok";
11
19
  ETError[ETError["Internal"] = 1] = "Internal";
12
20
  ETError[ETError["InvalidState"] = 2] = "InvalidState";
13
21
  ETError[ETError["EndOfMethod"] = 3] = "EndOfMethod";
22
+ // Logical errors
14
23
  ETError[ETError["NotSupported"] = 16] = "NotSupported";
15
24
  ETError[ETError["NotImplemented"] = 17] = "NotImplemented";
16
25
  ETError[ETError["InvalidArgument"] = 18] = "InvalidArgument";
17
26
  ETError[ETError["InvalidType"] = 19] = "InvalidType";
18
27
  ETError[ETError["OperatorMissing"] = 20] = "OperatorMissing";
28
+ // Resource errors
19
29
  ETError[ETError["NotFound"] = 32] = "NotFound";
20
30
  ETError[ETError["MemoryAllocationFailed"] = 33] = "MemoryAllocationFailed";
21
31
  ETError[ETError["AccessFailed"] = 34] = "AccessFailed";
22
32
  ETError[ETError["InvalidProgram"] = 35] = "InvalidProgram";
33
+ // Delegate errors
23
34
  ETError[ETError["DelegateInvalidCompatibility"] = 48] = "DelegateInvalidCompatibility";
24
35
  ETError[ETError["DelegateMemoryAllocationFailed"] = 49] = "DelegateMemoryAllocationFailed";
25
36
  ETError[ETError["DelegateInvalidHandle"] = 50] = "DelegateInvalidHandle";
@@ -30,9 +41,12 @@ export const getError = e => {
30
41
  if (e in ETError) return ETError[e];
31
42
  return ETError[ETError.UndefinedError];
32
43
  }
44
+
45
+ // try to extract number from message (can contain false positives)
33
46
  const error = e;
34
47
  const errorCode = parseInt(error.message, 10);
35
- if (errorCode in ETError) return ETError[errorCode];
36
- return ETError[ETError.UndefinedError];
48
+ const message = Number.isNaN(errorCode) ? error.message : ' ' + error.message.slice(`${errorCode}`.length).trimStart();
49
+ const ETErrorMessage = errorCode in ETError ? ETError[errorCode] : ETError[ETError.UndefinedError];
50
+ return ETErrorMessage + message;
37
51
  };
38
52
  //# sourceMappingURL=Error.js.map
@@ -1 +1 @@
1
- {"version":3,"names":["ETError","getError","e","UndefinedError","error","errorCode","parseInt","message"],"sourceRoot":"../../src","sources":["Error.ts"],"mappings":";;AAAA,WAAYA,OAAO,0BAAPA,OAAO;EAAPA,OAAO,CAAPA,OAAO;EAAPA,OAAO,CAAPA,OAAO;EAAPA,OAAO,CAAPA,OAAO;EAAPA,OAAO,CAAPA,OAAO;EAAPA,OAAO,CAAPA,OAAO;EAAPA,OAAO,CAAPA,OAAO;EAAPA,OAAO,CAAPA,OAAO;EAAPA,OAAO,CAAPA,OAAO;EAAPA,OAAO,CAAPA,OAAO;EAAPA,OAAO,CAAPA,OAAO;EAAPA,OAAO,CAAPA,OAAO;EAAPA,OAAO,CAAPA,OAAO;EAAPA,OAAO,CAAPA,OAAO;EAAPA,OAAO,CAAPA,OAAO;EAAPA,OAAO,CAAPA,OAAO;EAAPA,OAAO,CAAPA,OAAO;EAAPA,OAAO,CAAPA,OAAO;EAAPA,OAAO,CAAPA,OAAO;EAAPA,OAAO,CAAPA,OAAO;EAAPA,OAAO,CAAPA,OAAO;EAAPA,OAAO,CAAPA,OAAO;EAAPA,OAAO,CAAPA,OAAO;EAAA,OAAPA,OAAO;AAAA;AAoCnB,OAAO,MAAMC,QAAQ,GAAIC,CAAoB,IAAa;EACxD,IAAI,OAAOA,CAAC,KAAK,QAAQ,EAAE;IACzB,IAAIA,CAAC,IAAIF,OAAO,EAAE,OAAOA,OAAO,CAACE,CAAC,CAAC;IACnC,OAAOF,OAAO,CAACA,OAAO,CAACG,cAAc,CAAC;EACxC;EAEA,MAAMC,KAAK,GAAGF,CAAU;EACxB,MAAMG,SAAS,GAAGC,QAAQ,CAACF,KAAK,CAACG,OAAO,EAAE,EAAE,CAAC;EAC7C,IAAIF,SAAS,IAAIL,OAAO,EAAE,OAAOA,OAAO,CAACK,SAAS,CAAC;EACnD,OAAOL,OAAO,CAACA,OAAO,CAACG,cAAc,CAAC;AACxC,CAAC","ignoreList":[]}
1
+ {"version":3,"names":["ETError","getError","e","UndefinedError","error","errorCode","parseInt","message","Number","isNaN","slice","length","trimStart","ETErrorMessage"],"sourceRoot":"../../src","sources":["Error.ts"],"mappings":";;AAAA,WAAYA,OAAO,0BAAPA,OAAO;EACjB;EADUA,OAAO,CAAPA,OAAO;EAAPA,OAAO,CAAPA,OAAO;EAAPA,OAAO,CAAPA,OAAO;EAAPA,OAAO,CAAPA,OAAO;EAAPA,OAAO,CAAPA,OAAO;EAAPA,OAAO,CAAPA,OAAO;EASjB;EATUA,OAAO,CAAPA,OAAO;EAAPA,OAAO,CAAPA,OAAO;EAAPA,OAAO,CAAPA,OAAO;EAcjB;EACA;EACA;EAhBUA,OAAO,CAAPA,OAAO;EAAPA,OAAO,CAAPA,OAAO;EAAPA,OAAO,CAAPA,OAAO;EAAPA,OAAO,CAAPA,OAAO;EAsBjB;EAtBUA,OAAO,CAAPA,OAAO;EAAPA,OAAO,CAAPA,OAAO;EAAPA,OAAO,CAAPA,OAAO;EAAPA,OAAO,CAAPA,OAAO;EAAPA,OAAO,CAAPA,OAAO;EA6BjB;EA7BUA,OAAO,CAAPA,OAAO;EAAPA,OAAO,CAAPA,OAAO;EAAPA,OAAO,CAAPA,OAAO;EAAPA,OAAO,CAAPA,OAAO;EAmCjB;EAnCUA,OAAO,CAAPA,OAAO;EAAPA,OAAO,CAAPA,OAAO;EAAPA,OAAO,CAAPA,OAAO;EAAA,OAAPA,OAAO;AAAA;AAyCnB,OAAO,MAAMC,QAAQ,GAAIC,CAA4B,IAAa;EAChE,IAAI,OAAOA,CAAC,KAAK,QAAQ,EAAE;IACzB,IAAIA,CAAC,IAAIF,OAAO,EAAE,OAAOA,OAAO,CAACE,CAAC,CAAC;IACnC,OAAOF,OAAO,CAACA,OAAO,CAACG,cAAc,CAAC;EACxC;;EAEA;EACA,MAAMC,KAAK,GAAGF,CAAU;EACxB,MAAMG,SAAS,GAAGC,QAAQ,CAACF,KAAK,CAACG,OAAO,EAAE,EAAE,CAAC;EAC7C,MAAMA,OAAO,GAAGC,MAAM,CAACC,KAAK,CAACJ,SAAS,CAAC,GACnCD,KAAK,CAACG,OAAO,GACb,GAAG,GAAGH,KAAK,CAACG,OAAO,CAACG,KAAK,CAAC,GAAGL,SAAS,EAAE,CAACM,MAAM,CAAC,CAACC,SAAS,CAAC,CAAC;EAEhE,MAAMC,cAAc,GAClBR,SAAS,IAAIL,OAAO,GAAGA,OAAO,CAACK,SAAS,CAAC,GAAGL,OAAO,CAACA,OAAO,CAACG,cAAc,CACjE;EAEX,OAAOU,cAAc,GAAGN,OAAO;AACjC,CAAC","ignoreList":[]}