Package not found. Please check the package name and try again.

react-native-executorch 0.3.3 → 0.4.2

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (343) hide show
  1. package/README.md +30 -13
  2. package/android/build.gradle +1 -1
  3. package/android/src/main/java/com/swmansion/rnexecutorch/ETModule.kt +1 -2
  4. package/android/src/main/java/com/swmansion/rnexecutorch/ImageSegmentation.kt +58 -0
  5. package/android/src/main/java/com/swmansion/rnexecutorch/LLM.kt +13 -49
  6. package/android/src/main/java/com/swmansion/rnexecutorch/RnExecutorchPackage.kt +37 -0
  7. package/android/src/main/java/com/swmansion/rnexecutorch/StyleTransfer.kt +1 -1
  8. package/android/src/main/java/com/swmansion/rnexecutorch/TextEmbeddings.kt +51 -0
  9. package/android/src/main/java/com/swmansion/rnexecutorch/Tokenizer.kt +86 -0
  10. package/android/src/main/java/com/swmansion/rnexecutorch/models/BaseModel.kt +3 -4
  11. package/android/src/main/java/com/swmansion/rnexecutorch/models/TextEmbeddings/TextEmbeddingsModel.kt +48 -0
  12. package/android/src/main/java/com/swmansion/rnexecutorch/models/TextEmbeddings/TextEmbeddingsUtils.kt +37 -0
  13. package/android/src/main/java/com/swmansion/rnexecutorch/models/classification/ClassificationModel.kt +1 -0
  14. package/android/src/main/java/com/swmansion/rnexecutorch/models/imageSegmentation/Constants.kt +26 -0
  15. package/android/src/main/java/com/swmansion/rnexecutorch/models/imageSegmentation/ImageSegmentationModel.kt +142 -0
  16. package/android/src/main/java/com/swmansion/rnexecutorch/models/ocr/utils/RecognizerUtils.kt +3 -0
  17. package/android/src/main/java/com/swmansion/rnexecutorch/models/{StyleTransferModel.kt → styleTransfer/StyleTransferModel.kt} +2 -1
  18. package/android/src/main/java/com/swmansion/rnexecutorch/utils/ArrayUtils.kt +0 -8
  19. package/android/src/main/java/com/swmansion/rnexecutorch/{models/classification/Utils.kt → utils/Numerical.kt} +1 -1
  20. package/ios/ExecutorchLib.xcframework/Info.plist +4 -4
  21. package/ios/ExecutorchLib.xcframework/ios-arm64/ExecutorchLib.framework/ExecutorchLib +0 -0
  22. package/ios/ExecutorchLib.xcframework/ios-arm64/ExecutorchLib.framework/Headers/HuggingFaceTokenizer.h +14 -0
  23. package/ios/ExecutorchLib.xcframework/ios-arm64/ExecutorchLib.framework/Headers/LLaMARunner.h +1 -23
  24. package/ios/ExecutorchLib.xcframework/ios-arm64/ExecutorchLib.framework/Info.plist +0 -0
  25. package/ios/ExecutorchLib.xcframework/ios-arm64-simulator/ExecutorchLib.framework/ExecutorchLib +0 -0
  26. package/ios/ExecutorchLib.xcframework/ios-arm64-simulator/ExecutorchLib.framework/Headers/HuggingFaceTokenizer.h +14 -0
  27. package/ios/ExecutorchLib.xcframework/ios-arm64-simulator/ExecutorchLib.framework/Headers/LLaMARunner.h +1 -23
  28. package/ios/ExecutorchLib.xcframework/ios-arm64-simulator/ExecutorchLib.framework/Info.plist +0 -0
  29. package/ios/RnExecutorch/Classification.mm +15 -18
  30. package/ios/RnExecutorch/ETModule.mm +6 -5
  31. package/ios/RnExecutorch/ImageSegmentation.h +5 -0
  32. package/ios/RnExecutorch/ImageSegmentation.mm +60 -0
  33. package/ios/RnExecutorch/LLM.mm +12 -53
  34. package/ios/RnExecutorch/OCR.mm +39 -43
  35. package/ios/RnExecutorch/ObjectDetection.mm +20 -20
  36. package/ios/RnExecutorch/SpeechToText.mm +6 -7
  37. package/ios/RnExecutorch/StyleTransfer.mm +16 -19
  38. package/ios/RnExecutorch/TextEmbeddings.h +5 -0
  39. package/ios/RnExecutorch/TextEmbeddings.mm +62 -0
  40. package/ios/RnExecutorch/Tokenizer.h +5 -0
  41. package/ios/RnExecutorch/Tokenizer.mm +83 -0
  42. package/ios/RnExecutorch/VerticalOCR.mm +36 -36
  43. package/ios/RnExecutorch/models/BaseModel.h +2 -5
  44. package/ios/RnExecutorch/models/BaseModel.mm +5 -15
  45. package/ios/RnExecutorch/models/classification/ClassificationModel.mm +2 -3
  46. package/ios/RnExecutorch/models/classification/Constants.mm +0 -1
  47. package/ios/RnExecutorch/models/image_segmentation/Constants.h +4 -0
  48. package/ios/RnExecutorch/models/image_segmentation/Constants.mm +8 -0
  49. package/ios/RnExecutorch/models/image_segmentation/ImageSegmentationModel.h +10 -0
  50. package/ios/RnExecutorch/models/image_segmentation/ImageSegmentationModel.mm +146 -0
  51. package/ios/RnExecutorch/models/object_detection/SSDLiteLargeModel.mm +1 -2
  52. package/ios/RnExecutorch/models/ocr/Detector.h +0 -2
  53. package/ios/RnExecutorch/models/ocr/Detector.mm +2 -1
  54. package/ios/RnExecutorch/models/ocr/RecognitionHandler.h +5 -4
  55. package/ios/RnExecutorch/models/ocr/RecognitionHandler.mm +9 -26
  56. package/ios/RnExecutorch/models/ocr/Recognizer.mm +1 -2
  57. package/ios/RnExecutorch/models/ocr/VerticalDetector.h +0 -2
  58. package/ios/RnExecutorch/models/ocr/VerticalDetector.mm +2 -1
  59. package/ios/RnExecutorch/models/ocr/utils/OCRUtils.mm +0 -1
  60. package/ios/RnExecutorch/models/stt/Moonshine.mm +1 -6
  61. package/ios/RnExecutorch/models/stt/SpeechToTextBaseModel.mm +7 -11
  62. package/ios/RnExecutorch/models/stt/Whisper.mm +0 -5
  63. package/ios/RnExecutorch/models/{StyleTransferModel.h → style_transfer/StyleTransferModel.h} +1 -1
  64. package/ios/RnExecutorch/models/{StyleTransferModel.mm → style_transfer/StyleTransferModel.mm} +2 -3
  65. package/ios/RnExecutorch/models/text_embeddings/TextEmbeddingsModel.h +15 -0
  66. package/ios/RnExecutorch/models/text_embeddings/TextEmbeddingsModel.mm +45 -0
  67. package/ios/RnExecutorch/models/text_embeddings/TextEmbeddingsUtils.h +8 -0
  68. package/ios/RnExecutorch/models/text_embeddings/TextEmbeddingsUtils.mm +49 -0
  69. package/ios/RnExecutorch/utils/Conversions.h +15 -0
  70. package/ios/RnExecutorch/utils/ImageProcessor.h +0 -1
  71. package/ios/RnExecutorch/{models/classification/Utils.h → utils/Numerical.h} +0 -2
  72. package/ios/RnExecutorch/{models/classification/Utils.mm → utils/Numerical.mm} +0 -2
  73. package/ios/RnExecutorch/utils/ObjectDetectionUtils.mm +0 -2
  74. package/ios/RnExecutorch/utils/SFFT.mm +1 -1
  75. package/ios/RnExecutorch/utils/ScalarType.h +0 -2
  76. package/lib/module/Error.js +16 -2
  77. package/lib/module/Error.js.map +1 -1
  78. package/lib/module/constants/{llamaDefaults.js → llmDefaults.js} +7 -3
  79. package/lib/module/constants/llmDefaults.js.map +1 -0
  80. package/lib/module/constants/modelUrls.js +88 -27
  81. package/lib/module/constants/modelUrls.js.map +1 -1
  82. package/lib/module/constants/ocr/models.js +290 -0
  83. package/lib/module/constants/ocr/models.js.map +1 -0
  84. package/lib/module/constants/ocr/symbols.js +137 -2
  85. package/lib/module/constants/ocr/symbols.js.map +1 -1
  86. package/lib/module/constants/sttDefaults.js +50 -25
  87. package/lib/module/constants/sttDefaults.js.map +1 -1
  88. package/lib/module/controllers/LLMController.js +205 -0
  89. package/lib/module/controllers/LLMController.js.map +1 -0
  90. package/lib/module/controllers/OCRController.js +5 -10
  91. package/lib/module/controllers/OCRController.js.map +1 -1
  92. package/lib/module/controllers/SpeechToTextController.js +225 -122
  93. package/lib/module/controllers/SpeechToTextController.js.map +1 -1
  94. package/lib/module/controllers/VerticalOCRController.js +6 -10
  95. package/lib/module/controllers/VerticalOCRController.js.map +1 -1
  96. package/lib/module/hooks/computer_vision/useClassification.js +8 -23
  97. package/lib/module/hooks/computer_vision/useClassification.js.map +1 -1
  98. package/lib/module/hooks/computer_vision/useImageSegmentation.js +13 -0
  99. package/lib/module/hooks/computer_vision/useImageSegmentation.js.map +1 -0
  100. package/lib/module/hooks/computer_vision/useOCR.js +11 -6
  101. package/lib/module/hooks/computer_vision/useOCR.js.map +1 -1
  102. package/lib/module/hooks/computer_vision/useObjectDetection.js +8 -23
  103. package/lib/module/hooks/computer_vision/useObjectDetection.js.map +1 -1
  104. package/lib/module/hooks/computer_vision/useStyleTransfer.js +8 -23
  105. package/lib/module/hooks/computer_vision/useStyleTransfer.js.map +1 -1
  106. package/lib/module/hooks/computer_vision/useVerticalOCR.js +10 -7
  107. package/lib/module/hooks/computer_vision/useVerticalOCR.js.map +1 -1
  108. package/lib/module/hooks/general/useExecutorchModule.js +8 -36
  109. package/lib/module/hooks/general/useExecutorchModule.js.map +1 -1
  110. package/lib/module/hooks/natural_language_processing/useLLM.js +54 -63
  111. package/lib/module/hooks/natural_language_processing/useLLM.js.map +1 -1
  112. package/lib/module/hooks/natural_language_processing/useSpeechToText.js +15 -11
  113. package/lib/module/hooks/natural_language_processing/useSpeechToText.js.map +1 -1
  114. package/lib/module/hooks/natural_language_processing/useTextEmbeddings.js +14 -0
  115. package/lib/module/hooks/natural_language_processing/useTextEmbeddings.js.map +1 -0
  116. package/lib/module/hooks/natural_language_processing/useTokenizer.js +54 -0
  117. package/lib/module/hooks/natural_language_processing/useTokenizer.js.map +1 -0
  118. package/lib/module/hooks/useModule.js +18 -62
  119. package/lib/module/hooks/useModule.js.map +1 -1
  120. package/lib/module/index.js +16 -2
  121. package/lib/module/index.js.map +1 -1
  122. package/lib/module/modules/BaseModule.js +9 -10
  123. package/lib/module/modules/BaseModule.js.map +1 -1
  124. package/lib/module/modules/computer_vision/ClassificationModule.js +8 -5
  125. package/lib/module/modules/computer_vision/ClassificationModule.js.map +1 -1
  126. package/lib/module/modules/computer_vision/ImageSegmentationModule.js +28 -0
  127. package/lib/module/modules/computer_vision/ImageSegmentationModule.js.map +1 -0
  128. package/lib/module/modules/computer_vision/ObjectDetectionModule.js +8 -5
  129. package/lib/module/modules/computer_vision/ObjectDetectionModule.js.map +1 -1
  130. package/lib/module/modules/computer_vision/StyleTransferModule.js +8 -5
  131. package/lib/module/modules/computer_vision/StyleTransferModule.js.map +1 -1
  132. package/lib/module/modules/general/ExecutorchModule.js +8 -5
  133. package/lib/module/modules/general/ExecutorchModule.js.map +1 -1
  134. package/lib/module/modules/natural_language_processing/LLMModule.js +46 -27
  135. package/lib/module/modules/natural_language_processing/LLMModule.js.map +1 -1
  136. package/lib/module/modules/natural_language_processing/SpeechToTextModule.js +8 -5
  137. package/lib/module/modules/natural_language_processing/SpeechToTextModule.js.map +1 -1
  138. package/lib/module/modules/natural_language_processing/TextEmbeddingsModule.js +14 -0
  139. package/lib/module/modules/natural_language_processing/TextEmbeddingsModule.js.map +1 -0
  140. package/lib/module/modules/natural_language_processing/TokenizerModule.js +26 -0
  141. package/lib/module/modules/natural_language_processing/TokenizerModule.js.map +1 -0
  142. package/lib/module/native/NativeClassification.js.map +1 -1
  143. package/lib/module/native/NativeImageSegmentation.js +5 -0
  144. package/lib/module/native/NativeImageSegmentation.js.map +1 -0
  145. package/lib/module/native/NativeLLM.js.map +1 -1
  146. package/lib/module/native/NativeTextEmbeddings.js +5 -0
  147. package/lib/module/native/NativeTextEmbeddings.js.map +1 -0
  148. package/lib/module/native/NativeTokenizer.js +5 -0
  149. package/lib/module/native/NativeTokenizer.js.map +1 -0
  150. package/lib/module/native/RnExecutorchModules.js +18 -113
  151. package/lib/module/native/RnExecutorchModules.js.map +1 -1
  152. package/lib/module/types/common.js.map +1 -1
  153. package/lib/module/types/imageSegmentation.js +29 -0
  154. package/lib/module/types/imageSegmentation.js.map +1 -0
  155. package/lib/module/types/llm.js +7 -0
  156. package/lib/module/types/llm.js.map +1 -0
  157. package/lib/module/types/{object_detection.js → objectDetection.js} +1 -1
  158. package/lib/module/types/objectDetection.js.map +1 -0
  159. package/lib/module/types/ocr.js +2 -0
  160. package/lib/module/types/stt.js +82 -0
  161. package/lib/module/types/stt.js.map +1 -0
  162. package/lib/module/utils/ResourceFetcher.js +156 -0
  163. package/lib/module/utils/ResourceFetcher.js.map +1 -0
  164. package/lib/module/utils/llm.js +25 -0
  165. package/lib/module/utils/llm.js.map +1 -0
  166. package/lib/module/utils/stt.js +22 -0
  167. package/lib/module/utils/stt.js.map +1 -0
  168. package/lib/typescript/Error.d.ts +4 -1
  169. package/lib/typescript/Error.d.ts.map +1 -1
  170. package/lib/typescript/constants/{llamaDefaults.d.ts → llmDefaults.d.ts} +5 -5
  171. package/lib/typescript/constants/llmDefaults.d.ts.map +1 -0
  172. package/lib/typescript/constants/modelUrls.d.ts +74 -28
  173. package/lib/typescript/constants/modelUrls.d.ts.map +1 -1
  174. package/lib/typescript/constants/ocr/models.d.ts +285 -0
  175. package/lib/typescript/constants/ocr/models.d.ts.map +1 -0
  176. package/lib/typescript/constants/ocr/symbols.d.ts +73 -1
  177. package/lib/typescript/constants/ocr/symbols.d.ts.map +1 -1
  178. package/lib/typescript/constants/sttDefaults.d.ts +8 -13
  179. package/lib/typescript/constants/sttDefaults.d.ts.map +1 -1
  180. package/lib/typescript/controllers/LLMController.d.ts +46 -0
  181. package/lib/typescript/controllers/LLMController.d.ts.map +1 -0
  182. package/lib/typescript/controllers/OCRController.d.ts.map +1 -1
  183. package/lib/typescript/controllers/SpeechToTextController.d.ts +30 -16
  184. package/lib/typescript/controllers/SpeechToTextController.d.ts.map +1 -1
  185. package/lib/typescript/controllers/VerticalOCRController.d.ts +1 -1
  186. package/lib/typescript/controllers/VerticalOCRController.d.ts.map +1 -1
  187. package/lib/typescript/hooks/computer_vision/useClassification.d.ts +5 -5
  188. package/lib/typescript/hooks/computer_vision/useClassification.d.ts.map +1 -1
  189. package/lib/typescript/hooks/computer_vision/useImageSegmentation.d.ts +37 -0
  190. package/lib/typescript/hooks/computer_vision/useImageSegmentation.d.ts.map +1 -0
  191. package/lib/typescript/hooks/computer_vision/useOCR.d.ts +2 -1
  192. package/lib/typescript/hooks/computer_vision/useOCR.d.ts.map +1 -1
  193. package/lib/typescript/hooks/computer_vision/useObjectDetection.d.ts +5 -4
  194. package/lib/typescript/hooks/computer_vision/useObjectDetection.d.ts.map +1 -1
  195. package/lib/typescript/hooks/computer_vision/useStyleTransfer.d.ts +4 -2
  196. package/lib/typescript/hooks/computer_vision/useStyleTransfer.d.ts.map +1 -1
  197. package/lib/typescript/hooks/computer_vision/useVerticalOCR.d.ts +2 -1
  198. package/lib/typescript/hooks/computer_vision/useVerticalOCR.d.ts.map +1 -1
  199. package/lib/typescript/hooks/general/useExecutorchModule.d.ts +5 -6
  200. package/lib/typescript/hooks/general/useExecutorchModule.d.ts.map +1 -1
  201. package/lib/typescript/hooks/natural_language_processing/useLLM.d.ts +6 -6
  202. package/lib/typescript/hooks/natural_language_processing/useLLM.d.ts.map +1 -1
  203. package/lib/typescript/hooks/natural_language_processing/useSpeechToText.d.ts +7 -3
  204. package/lib/typescript/hooks/natural_language_processing/useSpeechToText.d.ts.map +1 -1
  205. package/lib/typescript/hooks/natural_language_processing/useTextEmbeddings.d.ts +13 -0
  206. package/lib/typescript/hooks/natural_language_processing/useTextEmbeddings.d.ts.map +1 -0
  207. package/lib/typescript/hooks/natural_language_processing/useTokenizer.d.ts +16 -0
  208. package/lib/typescript/hooks/natural_language_processing/useTokenizer.d.ts.map +1 -0
  209. package/lib/typescript/hooks/useModule.d.ts +11 -10
  210. package/lib/typescript/hooks/useModule.d.ts.map +1 -1
  211. package/lib/typescript/index.d.ts +15 -2
  212. package/lib/typescript/index.d.ts.map +1 -1
  213. package/lib/typescript/modules/BaseModule.d.ts +4 -5
  214. package/lib/typescript/modules/BaseModule.d.ts.map +1 -1
  215. package/lib/typescript/modules/computer_vision/ClassificationModule.d.ts +7 -7
  216. package/lib/typescript/modules/computer_vision/ClassificationModule.d.ts.map +1 -1
  217. package/lib/typescript/modules/computer_vision/ImageSegmentationModule.d.ts +32 -0
  218. package/lib/typescript/modules/computer_vision/ImageSegmentationModule.d.ts.map +1 -0
  219. package/lib/typescript/modules/computer_vision/ObjectDetectionModule.d.ts +6 -5
  220. package/lib/typescript/modules/computer_vision/ObjectDetectionModule.d.ts.map +1 -1
  221. package/lib/typescript/modules/computer_vision/StyleTransferModule.d.ts +6 -5
  222. package/lib/typescript/modules/computer_vision/StyleTransferModule.d.ts.map +1 -1
  223. package/lib/typescript/modules/general/ExecutorchModule.d.ts +4 -3
  224. package/lib/typescript/modules/general/ExecutorchModule.d.ts.map +1 -1
  225. package/lib/typescript/modules/natural_language_processing/LLMModule.d.ts +19 -5
  226. package/lib/typescript/modules/natural_language_processing/LLMModule.d.ts.map +1 -1
  227. package/lib/typescript/modules/natural_language_processing/SpeechToTextModule.d.ts +7 -4
  228. package/lib/typescript/modules/natural_language_processing/SpeechToTextModule.d.ts.map +1 -1
  229. package/lib/typescript/modules/natural_language_processing/TextEmbeddingsModule.d.ts +8 -0
  230. package/lib/typescript/modules/natural_language_processing/TextEmbeddingsModule.d.ts.map +1 -0
  231. package/lib/typescript/modules/natural_language_processing/TokenizerModule.d.ts +12 -0
  232. package/lib/typescript/modules/natural_language_processing/TokenizerModule.d.ts.map +1 -0
  233. package/lib/typescript/native/NativeClassification.d.ts.map +1 -1
  234. package/lib/typescript/native/NativeImageSegmentation.d.ts +10 -0
  235. package/lib/typescript/native/NativeImageSegmentation.d.ts.map +1 -0
  236. package/lib/typescript/native/NativeLLM.d.ts +3 -4
  237. package/lib/typescript/native/NativeLLM.d.ts.map +1 -1
  238. package/lib/typescript/native/NativeObjectDetection.d.ts +1 -1
  239. package/lib/typescript/native/NativeObjectDetection.d.ts.map +1 -1
  240. package/lib/typescript/native/NativeSpeechToText.d.ts +2 -2
  241. package/lib/typescript/native/NativeSpeechToText.d.ts.map +1 -1
  242. package/lib/typescript/native/NativeTextEmbeddings.d.ts +8 -0
  243. package/lib/typescript/native/NativeTextEmbeddings.d.ts.map +1 -0
  244. package/lib/typescript/native/NativeTokenizer.d.ts +12 -0
  245. package/lib/typescript/native/NativeTokenizer.d.ts.map +1 -0
  246. package/lib/typescript/native/RnExecutorchModules.d.ts +18 -41
  247. package/lib/typescript/native/RnExecutorchModules.d.ts.map +1 -1
  248. package/lib/typescript/types/common.d.ts +1 -26
  249. package/lib/typescript/types/common.d.ts.map +1 -1
  250. package/lib/typescript/types/imageSegmentation.d.ts +25 -0
  251. package/lib/typescript/types/imageSegmentation.d.ts.map +1 -0
  252. package/lib/typescript/types/llm.d.ts +38 -0
  253. package/lib/typescript/types/llm.d.ts.map +1 -0
  254. package/lib/typescript/types/{object_detection.d.ts → objectDetection.d.ts} +1 -1
  255. package/lib/typescript/types/objectDetection.d.ts.map +1 -0
  256. package/lib/typescript/types/ocr.d.ts +2 -1
  257. package/lib/typescript/types/ocr.d.ts.map +1 -1
  258. package/lib/typescript/types/stt.d.ts +91 -0
  259. package/lib/typescript/types/stt.d.ts.map +1 -0
  260. package/lib/typescript/utils/ResourceFetcher.d.ts +17 -0
  261. package/lib/typescript/utils/ResourceFetcher.d.ts.map +1 -0
  262. package/lib/typescript/utils/llm.d.ts +3 -0
  263. package/lib/typescript/utils/llm.d.ts.map +1 -0
  264. package/lib/typescript/utils/stt.d.ts +2 -0
  265. package/lib/typescript/utils/stt.d.ts.map +1 -0
  266. package/package.json +13 -49
  267. package/react-native-executorch.podspec +1 -1
  268. package/src/Error.ts +16 -3
  269. package/src/constants/llmDefaults.ts +14 -0
  270. package/src/constants/modelUrls.ts +146 -39
  271. package/src/constants/ocr/models.ts +453 -0
  272. package/src/constants/ocr/symbols.ts +147 -3
  273. package/src/constants/sttDefaults.ts +55 -37
  274. package/src/controllers/LLMController.ts +286 -0
  275. package/src/controllers/OCRController.ts +14 -28
  276. package/src/controllers/SpeechToTextController.ts +318 -180
  277. package/src/controllers/VerticalOCRController.ts +17 -32
  278. package/src/hooks/computer_vision/useClassification.ts +11 -26
  279. package/src/hooks/computer_vision/useImageSegmentation.ts +18 -0
  280. package/src/hooks/computer_vision/useOCR.ts +17 -5
  281. package/src/hooks/computer_vision/useObjectDetection.ts +10 -24
  282. package/src/hooks/computer_vision/useStyleTransfer.ts +9 -25
  283. package/src/hooks/computer_vision/useVerticalOCR.ts +11 -4
  284. package/src/hooks/general/useExecutorchModule.ts +10 -50
  285. package/src/hooks/natural_language_processing/useLLM.ts +80 -97
  286. package/src/hooks/natural_language_processing/useSpeechToText.ts +39 -12
  287. package/src/hooks/natural_language_processing/useTextEmbeddings.ts +18 -0
  288. package/src/hooks/natural_language_processing/useTokenizer.ts +61 -0
  289. package/src/hooks/useModule.ts +32 -92
  290. package/src/index.tsx +16 -2
  291. package/src/modules/BaseModule.ts +16 -26
  292. package/src/modules/computer_vision/ClassificationModule.ts +13 -8
  293. package/src/modules/computer_vision/ImageSegmentationModule.ts +39 -0
  294. package/src/modules/computer_vision/ObjectDetectionModule.ts +13 -8
  295. package/src/modules/computer_vision/StyleTransferModule.ts +13 -8
  296. package/src/modules/general/ExecutorchModule.ts +11 -6
  297. package/src/modules/natural_language_processing/LLMModule.ts +64 -51
  298. package/src/modules/natural_language_processing/SpeechToTextModule.ts +25 -10
  299. package/src/modules/natural_language_processing/TextEmbeddingsModule.ts +18 -0
  300. package/src/modules/natural_language_processing/TokenizerModule.ts +34 -0
  301. package/src/native/NativeClassification.ts +0 -1
  302. package/src/native/NativeImageSegmentation.ts +14 -0
  303. package/src/native/NativeLLM.ts +3 -10
  304. package/src/native/NativeObjectDetection.ts +1 -1
  305. package/src/native/NativeSpeechToText.ts +2 -2
  306. package/src/native/NativeTextEmbeddings.ts +9 -0
  307. package/src/native/NativeTokenizer.ts +13 -0
  308. package/src/native/RnExecutorchModules.ts +54 -234
  309. package/src/types/common.ts +1 -44
  310. package/src/types/imageSegmentation.ts +25 -0
  311. package/src/types/llm.ts +57 -0
  312. package/src/types/ocr.ts +3 -1
  313. package/src/types/stt.ts +93 -0
  314. package/src/utils/ResourceFetcher.ts +196 -0
  315. package/src/utils/llm.ts +34 -0
  316. package/src/utils/stt.ts +28 -0
  317. package/android/src/main/java/com/swmansion/rnexecutorch/utils/llms/ConversationManager.kt +0 -68
  318. package/ios/ExecutorchLib.xcframework/ios-arm64-simulator/ExecutorchLib.framework/_CodeSignature/CodeResources +0 -124
  319. package/ios/RnExecutorch/utils/llms/Constants.h +0 -6
  320. package/ios/RnExecutorch/utils/llms/Constants.mm +0 -23
  321. package/ios/RnExecutorch/utils/llms/ConversationManager.h +0 -26
  322. package/ios/RnExecutorch/utils/llms/ConversationManager.mm +0 -71
  323. package/lib/module/constants/llamaDefaults.js.map +0 -1
  324. package/lib/module/modules/computer_vision/BaseCVModule.js +0 -14
  325. package/lib/module/modules/computer_vision/BaseCVModule.js.map +0 -1
  326. package/lib/module/types/object_detection.js.map +0 -1
  327. package/lib/module/utils/fetchResource.js +0 -93
  328. package/lib/module/utils/fetchResource.js.map +0 -1
  329. package/lib/module/utils/listDownloadedResources.js +0 -13
  330. package/lib/module/utils/listDownloadedResources.js.map +0 -1
  331. package/lib/typescript/constants/llamaDefaults.d.ts.map +0 -1
  332. package/lib/typescript/modules/computer_vision/BaseCVModule.d.ts +0 -9
  333. package/lib/typescript/modules/computer_vision/BaseCVModule.d.ts.map +0 -1
  334. package/lib/typescript/types/object_detection.d.ts.map +0 -1
  335. package/lib/typescript/utils/fetchResource.d.ts +0 -3
  336. package/lib/typescript/utils/fetchResource.d.ts.map +0 -1
  337. package/lib/typescript/utils/listDownloadedResources.d.ts +0 -3
  338. package/lib/typescript/utils/listDownloadedResources.d.ts.map +0 -1
  339. package/src/constants/llamaDefaults.ts +0 -9
  340. package/src/modules/computer_vision/BaseCVModule.ts +0 -22
  341. package/src/utils/fetchResource.ts +0 -106
  342. package/src/utils/listDownloadedResources.ts +0 -12
  343. /package/src/types/{object_detection.ts → objectDetection.ts} +0 -0
@@ -0,0 +1,146 @@
1
+ #import "ImageSegmentationModel.h"
2
+ #import "../../utils/Conversions.h"
3
+ #import "../../utils/ImageProcessor.h"
4
+ #import "../../utils/Numerical.h"
5
+ #import "Constants.h"
6
+ #import <unordered_set>
7
+
8
+ @interface ImageSegmentationModel ()
9
+ - (NSArray *)preprocess:(cv::Mat &)input;
10
+ - (NSDictionary *)postprocess:(NSArray *)output
11
+ returnClasses:(NSArray *)classesOfInterest
12
+ resize:(BOOL)resize;
13
+ @end
14
+
15
+ @implementation ImageSegmentationModel {
16
+ cv::Size originalSize;
17
+ }
18
+
19
+ - (cv::Size)getModelImageSize {
20
+ NSArray *inputShape = [module getInputShape:@0];
21
+ NSNumber *widthNumber = inputShape.lastObject;
22
+ NSNumber *heightNumber = inputShape[inputShape.count - 2];
23
+
24
+ int height = [heightNumber intValue];
25
+ int width = [widthNumber intValue];
26
+
27
+ return cv::Size(height, width);
28
+ }
29
+
30
+ - (NSArray *)preprocess:(cv::Mat &)input {
31
+ originalSize = cv::Size(input.cols, input.rows);
32
+
33
+ cv::Size modelImageSize = [self getModelImageSize];
34
+ cv::Mat output;
35
+ cv::resize(input, output, modelImageSize);
36
+
37
+ NSArray *modelInput = [ImageProcessor matToNSArray:output];
38
+ return modelInput;
39
+ }
40
+
41
+ std::vector<cv::Mat> extractResults(NSArray *result, std::size_t numLabels,
42
+ cv::Size modelImageSize,
43
+ cv::Size originalSize, BOOL resize) {
44
+ std::size_t numModelPixels = modelImageSize.height * modelImageSize.width;
45
+
46
+ std::vector<cv::Mat> resizedLabelScores(numLabels);
47
+ for (std::size_t label = 0; label < numLabels; ++label) {
48
+ cv::Mat labelMat = cv::Mat(modelImageSize, CV_64F);
49
+
50
+ for (std::size_t pixel = 0; pixel < numModelPixels; ++pixel) {
51
+ int row = pixel / modelImageSize.width;
52
+ int col = pixel % modelImageSize.width;
53
+ labelMat.at<double>(row, col) =
54
+ [result[label * numModelPixels + pixel] doubleValue];
55
+ }
56
+
57
+ if (resize) {
58
+ cv::resize(labelMat, resizedLabelScores[label], originalSize);
59
+ } else {
60
+ resizedLabelScores[label] = std::move(labelMat);
61
+ }
62
+ }
63
+ return resizedLabelScores;
64
+ }
65
+
66
+ void adjustScoresPerPixel(std::vector<cv::Mat> &labelScores, cv::Mat &argMax,
67
+ cv::Size outputSize, std::size_t numLabels) {
68
+ std::size_t numOutputPixels = outputSize.height * outputSize.width;
69
+ for (std::size_t pixel = 0; pixel < numOutputPixels; ++pixel) {
70
+ int row = pixel / outputSize.width;
71
+ int col = pixel % outputSize.width;
72
+ std::vector<double> scores;
73
+ scores.reserve(numLabels);
74
+ for (const auto &mat : labelScores) {
75
+ scores.push_back(mat.at<double>(row, col));
76
+ }
77
+
78
+ std::vector<double> adjustedScores = softmax(scores);
79
+
80
+ for (std::size_t label = 0; label < numLabels; ++label) {
81
+ labelScores[label].at<double>(row, col) = adjustedScores[label];
82
+ }
83
+
84
+ auto maxIt = std::max_element(scores.begin(), scores.end());
85
+ argMax.at<int>(row, col) = std::distance(scores.begin(), maxIt);
86
+ }
87
+ }
88
+
89
+ - (NSDictionary *)postprocess:(NSArray *)output
90
+ returnClasses:(NSArray *)classesOfInterest
91
+ resize:(BOOL)resize {
92
+ cv::Size modelImageSize = [self getModelImageSize];
93
+
94
+ std::size_t numLabels = deeplabv3_resnet50_labels.size();
95
+
96
+ NSAssert((std::size_t)output.count ==
97
+ numLabels * modelImageSize.height * modelImageSize.width,
98
+ @"Model generated unexpected output size.");
99
+
100
+ // For each label extract it's matrix,
101
+ // and rescale it to the original size if `resize`
102
+ std::vector<cv::Mat> resizedLabelScores =
103
+ extractResults(output, numLabels, modelImageSize, originalSize, resize);
104
+
105
+ cv::Size outputSize = resize ? originalSize : modelImageSize;
106
+ cv::Mat argMax = cv::Mat(outputSize, CV_32S);
107
+
108
+ // For each pixel apply softmax across all the labels and calculate the argMax
109
+ adjustScoresPerPixel(resizedLabelScores, argMax, outputSize, numLabels);
110
+
111
+ std::unordered_set<std::string> labelSet;
112
+
113
+ for (id label in classesOfInterest) {
114
+ labelSet.insert(std::string([label UTF8String]));
115
+ }
116
+
117
+ NSMutableDictionary *result = [NSMutableDictionary dictionary];
118
+
119
+ // Convert to NSArray and populate the final dictionary
120
+ for (std::size_t label = 0; label < numLabels; ++label) {
121
+ if (labelSet.contains(deeplabv3_resnet50_labels[label])) {
122
+ NSString *labelString = @(deeplabv3_resnet50_labels[label].c_str());
123
+ NSArray *arr = simpleMatToNSArray<double>(resizedLabelScores[label]);
124
+ result[labelString] = arr;
125
+ }
126
+ }
127
+
128
+ result[@"ARGMAX"] = simpleMatToNSArray<int>(argMax);
129
+
130
+ return result;
131
+ }
132
+
133
+ - (NSDictionary *)runModel:(cv::Mat &)input
134
+ returnClasses:(NSArray *)classesOfInterest
135
+ resize:(BOOL)resize {
136
+ NSArray *modelInput = [self preprocess:input];
137
+ NSArray *result = [self forward:@[ modelInput ]];
138
+
139
+ NSDictionary *output = [self postprocess:result[0]
140
+ returnClasses:classesOfInterest
141
+ resize:resize];
142
+
143
+ return output;
144
+ }
145
+
146
+ @end
@@ -1,7 +1,6 @@
1
1
  #include "SSDLiteLargeModel.hpp"
2
2
  #include "../../utils/ObjectDetectionUtils.hpp"
3
3
  #include "ImageProcessor.h"
4
- #include <vector>
5
4
 
6
5
  float constexpr iouThreshold = 0.55;
7
6
  float constexpr detectionThreshold = 0.7;
@@ -54,7 +53,7 @@ int constexpr modelInputHeight = 320;
54
53
  int inputImageWidth = size.width;
55
54
  int inputImageHeight = size.height;
56
55
  NSArray *modelInput = [self preprocess:input];
57
- NSArray *forwardResult = [self forward:modelInput];
56
+ NSArray *forwardResult = [self forward:@[ modelInput ]];
58
57
  NSArray *output =
59
58
  [self postprocess:forwardResult
60
59
  widthRatio:inputImageWidth / (float)modelInputWidth
@@ -1,7 +1,5 @@
1
1
  #import "BaseModel.h"
2
- #import "RecognitionHandler.h"
3
2
  #import "opencv2/opencv.hpp"
4
- #import "utils/Constants.h"
5
3
 
6
4
  @interface Detector : BaseModel
7
5
 
@@ -1,5 +1,6 @@
1
1
  #import "Detector.h"
2
2
  #import "../../utils/ImageProcessor.h"
3
+ #import "utils/Constants.h"
3
4
  #import "utils/DetectorUtils.h"
4
5
  #import "utils/OCRUtils.h"
5
6
 
@@ -92,7 +93,7 @@
92
93
 
93
94
  - (NSArray *)runModel:(cv::Mat &)input {
94
95
  NSArray *modelInput = [self preprocess:input];
95
- NSArray *modelResult = [self forward:modelInput];
96
+ NSArray *modelResult = [self forward:@[ modelInput ]];
96
97
  NSArray *result = [self postprocess:modelResult];
97
98
  return result;
98
99
  }
@@ -3,10 +3,11 @@
3
3
  @interface RecognitionHandler : NSObject
4
4
 
5
5
  - (instancetype)initWithSymbols:(NSString *)symbols;
6
- - (void)loadRecognizers:(NSString *)largeRecognizerPath
7
- mediumRecognizerPath:(NSString *)mediumRecognizerPath
8
- smallRecognizerPath:(NSString *)smallRecognizerPath
9
- completion:(void (^)(BOOL, NSNumber *))completion;
6
+
7
+ - (NSNumber *)loadRecognizers:(NSString *)largeRecognizerPath
8
+ mediumRecognizerPath:(NSString *)mediumRecognizerPath
9
+ smallRecognizerPath:(NSString *)smallRecognizerPath;
10
+
10
11
  - (NSArray *)recognize:(NSArray<NSDictionary *> *)bBoxesList
11
12
  imgGray:(cv::Mat)imgGray
12
13
  desiredWidth:(int)desiredWidth
@@ -1,12 +1,9 @@
1
1
  #import "RecognitionHandler.h"
2
- #import "../../utils/ImageProcessor.h"
3
2
  #import "./utils/CTCLabelConverter.h"
4
3
  #import "./utils/Constants.h"
5
4
  #import "./utils/OCRUtils.h"
6
5
  #import "./utils/RecognizerUtils.h"
7
- #import "ExecutorchLib/ETModel.h"
8
6
  #import "Recognizer.h"
9
- #import <React/RCTBridgeModule.h>
10
7
 
11
8
  /*
12
9
  RecognitionHandler class is responsible for loading and choosing the
@@ -34,15 +31,12 @@
34
31
  return self;
35
32
  }
36
33
 
37
- - (void)loadRecognizers:(NSString *)largeRecognizerPath
38
- mediumRecognizerPath:(NSString *)mediumRecognizerPath
39
- smallRecognizerPath:(NSString *)smallRecognizerPath
40
- completion:(void (^)(BOOL, NSNumber *))completion {
41
- dispatch_group_t group = dispatch_group_create();
42
- __block BOOL allSuccessful = YES;
43
-
34
+ - (NSNumber *)loadRecognizers:(NSString *)largeRecognizerPath
35
+ mediumRecognizerPath:(NSString *)mediumRecognizerPath
36
+ smallRecognizerPath:(NSString *)smallRecognizerPath {
44
37
  NSArray<Recognizer *> *recognizers =
45
38
  @[ recognizerLarge, recognizerMedium, recognizerSmall ];
39
+
46
40
  NSArray<NSString *> *paths =
47
41
  @[ largeRecognizerPath, mediumRecognizerPath, smallRecognizerPath ];
48
42
 
@@ -50,24 +44,13 @@
50
44
  Recognizer *recognizer = recognizers[i];
51
45
  NSString *path = paths[i];
52
46
 
53
- dispatch_group_enter(group);
54
- [recognizer loadModel:[NSURL URLWithString:path]
55
- completion:^(BOOL success, NSNumber *errorCode) {
56
- if (!success) {
57
- allSuccessful = NO;
58
- dispatch_group_leave(group);
59
- completion(NO, errorCode);
60
- return;
61
- }
62
- dispatch_group_leave(group);
63
- }];
47
+ NSNumber *errorCode = [recognizer loadModel:path];
48
+ if ([errorCode intValue] != 0) {
49
+ return errorCode;
50
+ }
64
51
  }
65
52
 
66
- dispatch_group_notify(group, dispatch_get_main_queue(), ^{
67
- if (allSuccessful) {
68
- completion(YES, @(0));
69
- }
70
- });
53
+ return @0;
71
54
  }
72
55
 
73
56
  - (NSArray *)runModel:(cv::Mat)croppedImage {
@@ -1,7 +1,6 @@
1
1
  #import "Recognizer.h"
2
2
  #import "../../utils/ImageProcessor.h"
3
3
  #import "RecognizerUtils.h"
4
- #import "utils/OCRUtils.h"
5
4
 
6
5
  /*
7
6
  The model used as detector is based on CRNN paper.
@@ -69,7 +68,7 @@
69
68
 
70
69
  - (NSArray *)runModel:(cv::Mat &)input {
71
70
  NSArray *modelInput = [self preprocess:input];
72
- NSArray *modelResult = [self forward:modelInput];
71
+ NSArray *modelResult = [self forward:@[ modelInput ]];
73
72
  NSArray *result = [self postprocess:modelResult];
74
73
 
75
74
  return result;
@@ -1,7 +1,5 @@
1
1
  #import "BaseModel.h"
2
- #import "RecognitionHandler.h"
3
2
  #import "opencv2/opencv.hpp"
4
- #import "utils/Constants.h"
5
3
 
6
4
  @interface VerticalDetector : BaseModel
7
5
 
@@ -1,5 +1,6 @@
1
1
  #import "VerticalDetector.h"
2
2
  #import "../../utils/ImageProcessor.h"
3
+ #import "utils/Constants.h"
3
4
  #import "utils/DetectorUtils.h"
4
5
  #import "utils/OCRUtils.h"
5
6
 
@@ -109,7 +110,7 @@
109
110
 
110
111
  - (NSArray *)runModel:(cv::Mat &)input {
111
112
  NSArray *modelInput = [self preprocess:input];
112
- NSArray *modelResult = [self forward:modelInput];
113
+ NSArray *modelResult = [self forward:@[ modelInput ]];
113
114
  NSArray *result = [self postprocess:modelResult];
114
115
  return result;
115
116
  }
@@ -1,5 +1,4 @@
1
1
  #import "OCRUtils.h"
2
- #import "RecognizerUtils.h"
3
2
 
4
3
  @implementation OCRUtils
5
4
 
@@ -1,9 +1,4 @@
1
1
  #import "Moonshine.hpp"
2
- #import "ExecutorchLib/ETModel.h"
3
- #import "MoonshineEncoder.hpp"
4
- #import "Moonshinedecoder.hpp"
5
- #import <Foundation/Foundation.h>
6
- #import <UIKit/UIKit.h>
7
2
 
8
3
  @implementation Moonshine {
9
4
  MoonshineEncoder *encoder;
@@ -66,4 +61,4 @@
66
61
  }];
67
62
  }
68
63
 
69
- @end
64
+ @end
@@ -1,6 +1,4 @@
1
1
  #import "SpeechToTextBaseModel.hpp"
2
- #import <Foundation/Foundation.h>
3
- #import <UIKit/UIKit.h>
4
2
 
5
3
  @implementation SpeechToTextBaseModel
6
4
 
@@ -8,16 +6,14 @@
8
6
  withSource:(NSString *)source
9
7
  onSuccess:(void (^)(void))success
10
8
  onFailure:(void (^)(NSString *))failure {
9
+ NSNumber *errorCode = [model loadModel:source];
11
10
 
12
- [model loadModel:[NSURL URLWithString:source]
13
- completion:^(BOOL isSuccess, NSNumber *errorCode) {
14
- if (isSuccess) {
15
- success();
16
- } else {
17
- failure([NSString
18
- stringWithFormat:@"%ld", (long)[errorCode longValue]]);
19
- }
20
- }];
11
+ if ([errorCode intValue] != 0) {
12
+ failure([NSString stringWithFormat:@"%ld", (long)[errorCode longValue]]);
13
+ return;
14
+ }
15
+
16
+ success();
21
17
  }
22
18
 
23
19
  @end
@@ -1,11 +1,6 @@
1
1
  #import "Whisper.hpp"
2
- #import "../../utils/SFFT.hpp"
3
- #import "../../utils/ScalarType.h"
4
- #import "ExecutorchLib/ETModel.h"
5
2
  #import "WhisperEncoder.hpp"
6
3
  #import "Whisperdecoder.hpp"
7
- #import <Foundation/Foundation.h>
8
- #import <UIKit/UIKit.h>
9
4
 
10
5
  @implementation Whisper {
11
6
  WhisperEncoder *encoder;
@@ -1,4 +1,4 @@
1
- #import "BaseModel.h"
1
+ #import "../BaseModel.h"
2
2
  #import "opencv2/opencv.hpp"
3
3
 
4
4
  @interface StyleTransferModel : BaseModel
@@ -1,6 +1,5 @@
1
1
  #import "StyleTransferModel.h"
2
- #import "../utils/ImageProcessor.h"
3
- #import "opencv2/opencv.hpp"
2
+ #import "../../utils/ImageProcessor.h"
4
3
 
5
4
  @implementation StyleTransferModel {
6
5
  cv::Size originalSize;
@@ -42,7 +41,7 @@
42
41
 
43
42
  - (cv::Mat)runModel:(cv::Mat &)input {
44
43
  NSArray *modelInput = [self preprocess:input];
45
- NSArray *result = [self forward:modelInput];
44
+ NSArray *result = [self forward:@[ modelInput ]];
46
45
  input = [self postprocess:result[0]];
47
46
 
48
47
  return input;
@@ -0,0 +1,15 @@
1
+ #import "BaseModel.h"
2
+ #import "ExecutorchLib/HuggingFaceTokenizer.h"
3
+
4
+ @interface TextEmbeddingsModel : BaseModel {
5
+ @protected
6
+ HuggingFaceTokenizer *tokenizer;
7
+ }
8
+
9
+ - (void)loadTokenizer:(NSString *)tokenizerSource;
10
+ - (NSArray *)preprocess:(NSString *)input;
11
+ - (NSArray *)runModel:(NSString *)input;
12
+ - (NSArray *)postprocess:(NSArray *)input
13
+ attentionMask:(NSArray *)attentionMask;
14
+
15
+ @end
@@ -0,0 +1,45 @@
1
+ #import "TextEmbeddingsModel.h"
2
+ #import "TextEmbeddingsUtils.h"
3
+
4
+ @implementation TextEmbeddingsModel
5
+
6
+ - (NSArray *)preprocess:(NSString *)input {
7
+ NSArray *input_ids = [self->tokenizer encode:input];
8
+ NSMutableArray *attention_mask = [NSMutableArray new];
9
+ for (int i = 0; i < [input_ids count]; i++) {
10
+ [attention_mask addObject:@((int)([input_ids[i] intValue] != 0))];
11
+ }
12
+ return @[ input_ids, attention_mask ]; // [2, tokens]
13
+ }
14
+
15
+ - (NSArray *)postprocess:(NSArray *)modelOutput // [tokens * embedding_dim]
16
+ attentionMask:(NSArray *)attentionMask // [tokens]
17
+ {
18
+ NSArray *embeddings = [TextEmbeddingsUtils meanPooling:modelOutput
19
+ attentionMask:attentionMask];
20
+ return [TextEmbeddingsUtils normalize:embeddings];
21
+ }
22
+
23
+ - (NSArray *)runModel:(NSString *)input {
24
+ NSArray *modelInput = [self preprocess:input];
25
+
26
+ NSMutableArray *inputTypes = [NSMutableArray arrayWithObjects:@4, @4, nil];
27
+ NSMutableArray *shapes = [NSMutableArray new];
28
+
29
+ NSNumber *tokenCount = @([modelInput[0] count]);
30
+ for (__unused id _ in modelInput) {
31
+ [shapes addObject:[NSMutableArray arrayWithObjects:@1, tokenCount, nil]];
32
+ }
33
+
34
+ NSArray *modelOutput = [self forward:modelInput
35
+ shapes:shapes
36
+ inputTypes:inputTypes];
37
+ return [self postprocess:modelOutput[0] attentionMask:modelInput[1]];
38
+ }
39
+
40
+ - (void)loadTokenizer:(NSString *)tokenizerSource {
41
+ tokenizer =
42
+ [[HuggingFaceTokenizer alloc] initWithTokenizerPath:tokenizerSource];
43
+ }
44
+
45
+ @end
@@ -0,0 +1,8 @@
1
+ @interface TextEmbeddingsUtils : NSObject
2
+
3
+ + (NSArray *)meanPooling:(NSArray *)modelOutput
4
+ attentionMask:(NSArray *)attentionMask;
5
+
6
+ + (NSArray *)normalize:(NSArray *)embeddings;
7
+
8
+ @end
@@ -0,0 +1,49 @@
1
+ #import "TextEmbeddingsUtils.h"
2
+
3
+ @implementation TextEmbeddingsUtils
4
+
5
+ + (NSArray *)meanPooling:(NSArray *)modelOutput
6
+ attentionMask:(NSArray *)attentionMask {
7
+ NSInteger modelOutputLength = [modelOutput count];
8
+ NSInteger attentionMaskLength = [attentionMask count];
9
+ NSInteger embeddingDim = modelOutputLength / attentionMaskLength;
10
+
11
+ NSMutableArray *result = [NSMutableArray arrayWithCapacity:embeddingDim];
12
+
13
+ double sumMask = 0.0;
14
+ for (NSNumber *maskValue in attentionMask) {
15
+ sumMask += [maskValue intValue];
16
+ }
17
+ sumMask = fmax(sumMask, 1e-9);
18
+
19
+ for (NSInteger i = 0; i < embeddingDim; i++) {
20
+ double sum = 0;
21
+ for (NSInteger j = 0; j < attentionMaskLength; j++) {
22
+ sum += [modelOutput[j * embeddingDim + i] doubleValue] *
23
+ [attentionMask[j] intValue];
24
+ }
25
+ [result addObject:@(sum / sumMask)];
26
+ }
27
+
28
+ return result;
29
+ }
30
+
31
+ + (NSArray *)normalize:(NSArray *)embeddings {
32
+ NSInteger embeddingDim = [embeddings count];
33
+ double sum = 0.0;
34
+
35
+ for (NSNumber *value in embeddings) {
36
+ sum += [value doubleValue] * [value doubleValue];
37
+ }
38
+ sum = fmax(sqrt(sum), 1e-9);
39
+
40
+ NSMutableArray *normalizedResult =
41
+ [NSMutableArray arrayWithCapacity:embeddingDim];
42
+ for (NSNumber *value in embeddings) {
43
+ [normalizedResult addObject:@([value doubleValue] / sum)];
44
+ }
45
+
46
+ return normalizedResult;
47
+ }
48
+
49
+ @end
@@ -0,0 +1,15 @@
1
+ #import "opencv2/opencv.hpp"
2
+
3
+ // Convert a matrix containing a single value per cell to a NSArray
4
+ template <typename T>
5
+ NSArray* simpleMatToNSArray(const cv::Mat& mat) {
6
+ std::size_t numPixels = mat.rows * mat.cols;
7
+ NSMutableArray *arr = [[NSMutableArray alloc] initWithCapacity:numPixels];
8
+
9
+ for (std::size_t x = 0; x < mat.rows; ++x) {
10
+ for (std::size_t y = 0; y < mat.cols; ++y) {
11
+ arr[x * mat.cols + y] = @(mat.at<T>(x, y));
12
+ }
13
+ }
14
+ return arr;
15
+ }
@@ -1,4 +1,3 @@
1
- #import <UIKit/UIKit.h>
2
1
  #import <opencv2/opencv.hpp>
3
2
 
4
3
  @interface ImageProcessor : NSObject
@@ -1,3 +1 @@
1
- #include <vector>
2
-
3
1
  std::vector<double> softmax(const std::vector<double> &v);
@@ -1,6 +1,4 @@
1
1
  #include "Utils.h"
2
- #include <cmath>
3
- #include <vector>
4
2
 
5
3
  std::vector<double> softmax(const std::vector<double> &v) {
6
4
  std::vector<double> result(v.size());
@@ -1,7 +1,5 @@
1
1
  #include "ObjectDetectionUtils.hpp"
2
2
  #include "Constants.h"
3
- #include <map>
4
- #include <vector>
5
3
 
6
4
  NSString *floatLabelToNSString(float label) {
7
5
  int intLabel = static_cast<int>(label);
@@ -68,4 +68,4 @@
68
68
  free(a.imagp);
69
69
  }
70
70
 
71
- @end
71
+ @end
@@ -1,8 +1,6 @@
1
1
  #ifndef ScalarType_h
2
2
  #define ScalarType_h
3
3
 
4
- #import <Foundation/Foundation.h>
5
-
6
4
  @interface ScalarType : NSObject
7
5
 
8
6
  @property(class, nonatomic, readonly) NSNumber *Int8;
@@ -1,25 +1,36 @@
1
1
  "use strict";
2
2
 
3
3
  export let ETError = /*#__PURE__*/function (ETError) {
4
+ // React-native-ExecuTorch errors
4
5
  ETError[ETError["UndefinedError"] = 101] = "UndefinedError";
5
6
  ETError[ETError["ModuleNotLoaded"] = 102] = "ModuleNotLoaded";
6
7
  ETError[ETError["FileWriteFailed"] = 103] = "FileWriteFailed";
7
8
  ETError[ETError["ModelGenerating"] = 104] = "ModelGenerating";
8
9
  ETError[ETError["LanguageNotSupported"] = 105] = "LanguageNotSupported";
9
10
  ETError[ETError["InvalidModelSource"] = 255] = "InvalidModelSource";
11
+ //SpeechToText errors
12
+ ETError[ETError["MultilingualConfiguration"] = 160] = "MultilingualConfiguration";
13
+ ETError[ETError["MissingDataChunk"] = 161] = "MissingDataChunk";
14
+ ETError[ETError["StreamingNotStarted"] = 162] = "StreamingNotStarted";
15
+ // ExecuTorch mapped errors
16
+ // Based on: https://github.com/pytorch/executorch/blob/main/runtime/core/error.h
17
+ // System errors
10
18
  ETError[ETError["Ok"] = 0] = "Ok";
11
19
  ETError[ETError["Internal"] = 1] = "Internal";
12
20
  ETError[ETError["InvalidState"] = 2] = "InvalidState";
13
21
  ETError[ETError["EndOfMethod"] = 3] = "EndOfMethod";
22
+ // Logical errors
14
23
  ETError[ETError["NotSupported"] = 16] = "NotSupported";
15
24
  ETError[ETError["NotImplemented"] = 17] = "NotImplemented";
16
25
  ETError[ETError["InvalidArgument"] = 18] = "InvalidArgument";
17
26
  ETError[ETError["InvalidType"] = 19] = "InvalidType";
18
27
  ETError[ETError["OperatorMissing"] = 20] = "OperatorMissing";
28
+ // Resource errors
19
29
  ETError[ETError["NotFound"] = 32] = "NotFound";
20
30
  ETError[ETError["MemoryAllocationFailed"] = 33] = "MemoryAllocationFailed";
21
31
  ETError[ETError["AccessFailed"] = 34] = "AccessFailed";
22
32
  ETError[ETError["InvalidProgram"] = 35] = "InvalidProgram";
33
+ // Delegate errors
23
34
  ETError[ETError["DelegateInvalidCompatibility"] = 48] = "DelegateInvalidCompatibility";
24
35
  ETError[ETError["DelegateMemoryAllocationFailed"] = 49] = "DelegateMemoryAllocationFailed";
25
36
  ETError[ETError["DelegateInvalidHandle"] = 50] = "DelegateInvalidHandle";
@@ -30,9 +41,12 @@ export const getError = e => {
30
41
  if (e in ETError) return ETError[e];
31
42
  return ETError[ETError.UndefinedError];
32
43
  }
44
+
45
+ // try to extract number from message (can contain false positives)
33
46
  const error = e;
34
47
  const errorCode = parseInt(error.message, 10);
35
- if (errorCode in ETError) return ETError[errorCode];
36
- return ETError[ETError.UndefinedError];
48
+ const message = Number.isNaN(errorCode) ? error.message : ' ' + error.message.slice(`${errorCode}`.length).trimStart();
49
+ const ETErrorMessage = errorCode in ETError ? ETError[errorCode] : ETError[ETError.UndefinedError];
50
+ return ETErrorMessage + message;
37
51
  };
38
52
  //# sourceMappingURL=Error.js.map
@@ -1 +1 @@
1
- {"version":3,"names":["ETError","getError","e","UndefinedError","error","errorCode","parseInt","message"],"sourceRoot":"../../src","sources":["Error.ts"],"mappings":";;AAAA,WAAYA,OAAO,0BAAPA,OAAO;EAAPA,OAAO,CAAPA,OAAO;EAAPA,OAAO,CAAPA,OAAO;EAAPA,OAAO,CAAPA,OAAO;EAAPA,OAAO,CAAPA,OAAO;EAAPA,OAAO,CAAPA,OAAO;EAAPA,OAAO,CAAPA,OAAO;EAAPA,OAAO,CAAPA,OAAO;EAAPA,OAAO,CAAPA,OAAO;EAAPA,OAAO,CAAPA,OAAO;EAAPA,OAAO,CAAPA,OAAO;EAAPA,OAAO,CAAPA,OAAO;EAAPA,OAAO,CAAPA,OAAO;EAAPA,OAAO,CAAPA,OAAO;EAAPA,OAAO,CAAPA,OAAO;EAAPA,OAAO,CAAPA,OAAO;EAAPA,OAAO,CAAPA,OAAO;EAAPA,OAAO,CAAPA,OAAO;EAAPA,OAAO,CAAPA,OAAO;EAAPA,OAAO,CAAPA,OAAO;EAAPA,OAAO,CAAPA,OAAO;EAAPA,OAAO,CAAPA,OAAO;EAAPA,OAAO,CAAPA,OAAO;EAAA,OAAPA,OAAO;AAAA;AAoCnB,OAAO,MAAMC,QAAQ,GAAIC,CAAoB,IAAa;EACxD,IAAI,OAAOA,CAAC,KAAK,QAAQ,EAAE;IACzB,IAAIA,CAAC,IAAIF,OAAO,EAAE,OAAOA,OAAO,CAACE,CAAC,CAAC;IACnC,OAAOF,OAAO,CAACA,OAAO,CAACG,cAAc,CAAC;EACxC;EAEA,MAAMC,KAAK,GAAGF,CAAU;EACxB,MAAMG,SAAS,GAAGC,QAAQ,CAACF,KAAK,CAACG,OAAO,EAAE,EAAE,CAAC;EAC7C,IAAIF,SAAS,IAAIL,OAAO,EAAE,OAAOA,OAAO,CAACK,SAAS,CAAC;EACnD,OAAOL,OAAO,CAACA,OAAO,CAACG,cAAc,CAAC;AACxC,CAAC","ignoreList":[]}
1
+ {"version":3,"names":["ETError","getError","e","UndefinedError","error","errorCode","parseInt","message","Number","isNaN","slice","length","trimStart","ETErrorMessage"],"sourceRoot":"../../src","sources":["Error.ts"],"mappings":";;AAAA,WAAYA,OAAO,0BAAPA,OAAO;EACjB;EADUA,OAAO,CAAPA,OAAO;EAAPA,OAAO,CAAPA,OAAO;EAAPA,OAAO,CAAPA,OAAO;EAAPA,OAAO,CAAPA,OAAO;EAAPA,OAAO,CAAPA,OAAO;EAAPA,OAAO,CAAPA,OAAO;EASjB;EATUA,OAAO,CAAPA,OAAO;EAAPA,OAAO,CAAPA,OAAO;EAAPA,OAAO,CAAPA,OAAO;EAcjB;EACA;EACA;EAhBUA,OAAO,CAAPA,OAAO;EAAPA,OAAO,CAAPA,OAAO;EAAPA,OAAO,CAAPA,OAAO;EAAPA,OAAO,CAAPA,OAAO;EAsBjB;EAtBUA,OAAO,CAAPA,OAAO;EAAPA,OAAO,CAAPA,OAAO;EAAPA,OAAO,CAAPA,OAAO;EAAPA,OAAO,CAAPA,OAAO;EAAPA,OAAO,CAAPA,OAAO;EA6BjB;EA7BUA,OAAO,CAAPA,OAAO;EAAPA,OAAO,CAAPA,OAAO;EAAPA,OAAO,CAAPA,OAAO;EAAPA,OAAO,CAAPA,OAAO;EAmCjB;EAnCUA,OAAO,CAAPA,OAAO;EAAPA,OAAO,CAAPA,OAAO;EAAPA,OAAO,CAAPA,OAAO;EAAA,OAAPA,OAAO;AAAA;AAyCnB,OAAO,MAAMC,QAAQ,GAAIC,CAA4B,IAAa;EAChE,IAAI,OAAOA,CAAC,KAAK,QAAQ,EAAE;IACzB,IAAIA,CAAC,IAAIF,OAAO,EAAE,OAAOA,OAAO,CAACE,CAAC,CAAC;IACnC,OAAOF,OAAO,CAACA,OAAO,CAACG,cAAc,CAAC;EACxC;;EAEA;EACA,MAAMC,KAAK,GAAGF,CAAU;EACxB,MAAMG,SAAS,GAAGC,QAAQ,CAACF,KAAK,CAACG,OAAO,EAAE,EAAE,CAAC;EAC7C,MAAMA,OAAO,GAAGC,MAAM,CAACC,KAAK,CAACJ,SAAS,CAAC,GACnCD,KAAK,CAACG,OAAO,GACb,GAAG,GAAGH,KAAK,CAACG,OAAO,CAACG,KAAK,CAAC,GAAGL,SAAS,EAAE,CAACM,MAAM,CAAC,CAACC,SAAS,CAAC,CAAC;EAEhE,MAAMC,cAAc,GAClBR,SAAS,IAAIL,OAAO,GAAGA,OAAO,CAACK,SAAS,CAAC,GAAGL,OAAO,CAACA,OAAO,CAACG,cAAc,CACjE;EAEX,OAAOU,cAAc,GAAGN,OAAO;AACjC,CAAC","ignoreList":[]}