react-native-executorch 0.3.3 → 0.4.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +30 -13
- package/android/build.gradle +1 -1
- package/android/libs/executorch.aar +0 -0
- package/android/src/main/java/com/swmansion/rnexecutorch/ETModule.kt +1 -2
- package/android/src/main/java/com/swmansion/rnexecutorch/ImageSegmentation.kt +58 -0
- package/android/src/main/java/com/swmansion/rnexecutorch/LLM.kt +13 -49
- package/android/src/main/java/com/swmansion/rnexecutorch/RnExecutorchPackage.kt +37 -0
- package/android/src/main/java/com/swmansion/rnexecutorch/StyleTransfer.kt +1 -1
- package/android/src/main/java/com/swmansion/rnexecutorch/TextEmbeddings.kt +51 -0
- package/android/src/main/java/com/swmansion/rnexecutorch/Tokenizer.kt +86 -0
- package/android/src/main/java/com/swmansion/rnexecutorch/models/BaseModel.kt +3 -4
- package/android/src/main/java/com/swmansion/rnexecutorch/models/TextEmbeddings/TextEmbeddingsModel.kt +48 -0
- package/android/src/main/java/com/swmansion/rnexecutorch/models/TextEmbeddings/TextEmbeddingsUtils.kt +37 -0
- package/android/src/main/java/com/swmansion/rnexecutorch/models/classification/ClassificationModel.kt +1 -0
- package/android/src/main/java/com/swmansion/rnexecutorch/models/imageSegmentation/Constants.kt +26 -0
- package/android/src/main/java/com/swmansion/rnexecutorch/models/imageSegmentation/ImageSegmentationModel.kt +142 -0
- package/android/src/main/java/com/swmansion/rnexecutorch/models/ocr/utils/RecognizerUtils.kt +3 -0
- package/android/src/main/java/com/swmansion/rnexecutorch/models/{StyleTransferModel.kt → styleTransfer/StyleTransferModel.kt} +2 -1
- package/android/src/main/java/com/swmansion/rnexecutorch/utils/ArrayUtils.kt +0 -8
- package/android/src/main/java/com/swmansion/rnexecutorch/{models/classification/Utils.kt → utils/Numerical.kt} +1 -1
- package/ios/ExecutorchLib.xcframework/Info.plist +4 -4
- package/ios/ExecutorchLib.xcframework/ios-arm64/ExecutorchLib.framework/ExecutorchLib +0 -0
- package/ios/ExecutorchLib.xcframework/ios-arm64/ExecutorchLib.framework/Headers/HuggingFaceTokenizer.h +14 -0
- package/ios/ExecutorchLib.xcframework/ios-arm64/ExecutorchLib.framework/Headers/LLaMARunner.h +1 -23
- package/ios/ExecutorchLib.xcframework/ios-arm64/ExecutorchLib.framework/Info.plist +0 -0
- package/ios/ExecutorchLib.xcframework/ios-arm64-simulator/ExecutorchLib.framework/ExecutorchLib +0 -0
- package/ios/ExecutorchLib.xcframework/ios-arm64-simulator/ExecutorchLib.framework/Headers/HuggingFaceTokenizer.h +14 -0
- package/ios/ExecutorchLib.xcframework/ios-arm64-simulator/ExecutorchLib.framework/Headers/LLaMARunner.h +1 -23
- package/ios/ExecutorchLib.xcframework/ios-arm64-simulator/ExecutorchLib.framework/Info.plist +0 -0
- package/ios/RnExecutorch/Classification.mm +15 -18
- package/ios/RnExecutorch/ETModule.mm +6 -5
- package/ios/RnExecutorch/ImageSegmentation.h +5 -0
- package/ios/RnExecutorch/ImageSegmentation.mm +60 -0
- package/ios/RnExecutorch/LLM.mm +12 -53
- package/ios/RnExecutorch/OCR.mm +39 -43
- package/ios/RnExecutorch/ObjectDetection.mm +20 -20
- package/ios/RnExecutorch/SpeechToText.mm +6 -7
- package/ios/RnExecutorch/StyleTransfer.mm +16 -19
- package/ios/RnExecutorch/TextEmbeddings.h +5 -0
- package/ios/RnExecutorch/TextEmbeddings.mm +62 -0
- package/ios/RnExecutorch/Tokenizer.h +5 -0
- package/ios/RnExecutorch/Tokenizer.mm +83 -0
- package/ios/RnExecutorch/VerticalOCR.mm +36 -36
- package/ios/RnExecutorch/models/BaseModel.h +2 -5
- package/ios/RnExecutorch/models/BaseModel.mm +5 -15
- package/ios/RnExecutorch/models/classification/ClassificationModel.mm +2 -3
- package/ios/RnExecutorch/models/classification/Constants.mm +0 -1
- package/ios/RnExecutorch/models/image_segmentation/Constants.h +4 -0
- package/ios/RnExecutorch/models/image_segmentation/Constants.mm +8 -0
- package/ios/RnExecutorch/models/image_segmentation/ImageSegmentationModel.h +10 -0
- package/ios/RnExecutorch/models/image_segmentation/ImageSegmentationModel.mm +146 -0
- package/ios/RnExecutorch/models/object_detection/SSDLiteLargeModel.mm +1 -2
- package/ios/RnExecutorch/models/ocr/Detector.h +0 -2
- package/ios/RnExecutorch/models/ocr/Detector.mm +2 -1
- package/ios/RnExecutorch/models/ocr/RecognitionHandler.h +5 -4
- package/ios/RnExecutorch/models/ocr/RecognitionHandler.mm +9 -26
- package/ios/RnExecutorch/models/ocr/Recognizer.mm +1 -2
- package/ios/RnExecutorch/models/ocr/VerticalDetector.h +0 -2
- package/ios/RnExecutorch/models/ocr/VerticalDetector.mm +2 -1
- package/ios/RnExecutorch/models/ocr/utils/OCRUtils.mm +0 -1
- package/ios/RnExecutorch/models/stt/Moonshine.mm +1 -6
- package/ios/RnExecutorch/models/stt/SpeechToTextBaseModel.mm +7 -11
- package/ios/RnExecutorch/models/stt/Whisper.mm +0 -5
- package/ios/RnExecutorch/models/{StyleTransferModel.h → style_transfer/StyleTransferModel.h} +1 -1
- package/ios/RnExecutorch/models/{StyleTransferModel.mm → style_transfer/StyleTransferModel.mm} +2 -3
- package/ios/RnExecutorch/models/text_embeddings/TextEmbeddingsModel.h +15 -0
- package/ios/RnExecutorch/models/text_embeddings/TextEmbeddingsModel.mm +45 -0
- package/ios/RnExecutorch/models/text_embeddings/TextEmbeddingsUtils.h +8 -0
- package/ios/RnExecutorch/models/text_embeddings/TextEmbeddingsUtils.mm +49 -0
- package/ios/RnExecutorch/utils/Conversions.h +15 -0
- package/ios/RnExecutorch/utils/ImageProcessor.h +0 -1
- package/ios/RnExecutorch/{models/classification/Utils.h → utils/Numerical.h} +0 -2
- package/ios/RnExecutorch/{models/classification/Utils.mm → utils/Numerical.mm} +0 -2
- package/ios/RnExecutorch/utils/ObjectDetectionUtils.mm +0 -2
- package/ios/RnExecutorch/utils/SFFT.mm +1 -1
- package/ios/RnExecutorch/utils/ScalarType.h +0 -2
- package/lib/module/Error.js +16 -2
- package/lib/module/Error.js.map +1 -1
- package/lib/module/constants/{llamaDefaults.js → llmDefaults.js} +7 -3
- package/lib/module/constants/llmDefaults.js.map +1 -0
- package/lib/module/constants/modelUrls.js +88 -27
- package/lib/module/constants/modelUrls.js.map +1 -1
- package/lib/module/constants/ocr/models.js +290 -0
- package/lib/module/constants/ocr/models.js.map +1 -0
- package/lib/module/constants/ocr/symbols.js +137 -2
- package/lib/module/constants/ocr/symbols.js.map +1 -1
- package/lib/module/constants/sttDefaults.js +50 -25
- package/lib/module/constants/sttDefaults.js.map +1 -1
- package/lib/module/controllers/LLMController.js +205 -0
- package/lib/module/controllers/LLMController.js.map +1 -0
- package/lib/module/controllers/OCRController.js +5 -10
- package/lib/module/controllers/OCRController.js.map +1 -1
- package/lib/module/controllers/SpeechToTextController.js +225 -122
- package/lib/module/controllers/SpeechToTextController.js.map +1 -1
- package/lib/module/controllers/VerticalOCRController.js +6 -10
- package/lib/module/controllers/VerticalOCRController.js.map +1 -1
- package/lib/module/hooks/computer_vision/useClassification.js +8 -23
- package/lib/module/hooks/computer_vision/useClassification.js.map +1 -1
- package/lib/module/hooks/computer_vision/useImageSegmentation.js +13 -0
- package/lib/module/hooks/computer_vision/useImageSegmentation.js.map +1 -0
- package/lib/module/hooks/computer_vision/useOCR.js +11 -6
- package/lib/module/hooks/computer_vision/useOCR.js.map +1 -1
- package/lib/module/hooks/computer_vision/useObjectDetection.js +8 -23
- package/lib/module/hooks/computer_vision/useObjectDetection.js.map +1 -1
- package/lib/module/hooks/computer_vision/useStyleTransfer.js +8 -23
- package/lib/module/hooks/computer_vision/useStyleTransfer.js.map +1 -1
- package/lib/module/hooks/computer_vision/useVerticalOCR.js +10 -7
- package/lib/module/hooks/computer_vision/useVerticalOCR.js.map +1 -1
- package/lib/module/hooks/general/useExecutorchModule.js +8 -36
- package/lib/module/hooks/general/useExecutorchModule.js.map +1 -1
- package/lib/module/hooks/natural_language_processing/useLLM.js +54 -63
- package/lib/module/hooks/natural_language_processing/useLLM.js.map +1 -1
- package/lib/module/hooks/natural_language_processing/useSpeechToText.js +15 -11
- package/lib/module/hooks/natural_language_processing/useSpeechToText.js.map +1 -1
- package/lib/module/hooks/natural_language_processing/useTextEmbeddings.js +14 -0
- package/lib/module/hooks/natural_language_processing/useTextEmbeddings.js.map +1 -0
- package/lib/module/hooks/natural_language_processing/useTokenizer.js +54 -0
- package/lib/module/hooks/natural_language_processing/useTokenizer.js.map +1 -0
- package/lib/module/hooks/useModule.js +18 -62
- package/lib/module/hooks/useModule.js.map +1 -1
- package/lib/module/index.js +16 -2
- package/lib/module/index.js.map +1 -1
- package/lib/module/modules/BaseModule.js +9 -10
- package/lib/module/modules/BaseModule.js.map +1 -1
- package/lib/module/modules/computer_vision/ClassificationModule.js +8 -5
- package/lib/module/modules/computer_vision/ClassificationModule.js.map +1 -1
- package/lib/module/modules/computer_vision/ImageSegmentationModule.js +28 -0
- package/lib/module/modules/computer_vision/ImageSegmentationModule.js.map +1 -0
- package/lib/module/modules/computer_vision/ObjectDetectionModule.js +8 -5
- package/lib/module/modules/computer_vision/ObjectDetectionModule.js.map +1 -1
- package/lib/module/modules/computer_vision/StyleTransferModule.js +8 -5
- package/lib/module/modules/computer_vision/StyleTransferModule.js.map +1 -1
- package/lib/module/modules/general/ExecutorchModule.js +8 -5
- package/lib/module/modules/general/ExecutorchModule.js.map +1 -1
- package/lib/module/modules/natural_language_processing/LLMModule.js +46 -27
- package/lib/module/modules/natural_language_processing/LLMModule.js.map +1 -1
- package/lib/module/modules/natural_language_processing/SpeechToTextModule.js +8 -5
- package/lib/module/modules/natural_language_processing/SpeechToTextModule.js.map +1 -1
- package/lib/module/modules/natural_language_processing/TextEmbeddingsModule.js +14 -0
- package/lib/module/modules/natural_language_processing/TextEmbeddingsModule.js.map +1 -0
- package/lib/module/modules/natural_language_processing/TokenizerModule.js +26 -0
- package/lib/module/modules/natural_language_processing/TokenizerModule.js.map +1 -0
- package/lib/module/native/NativeClassification.js.map +1 -1
- package/lib/module/native/NativeImageSegmentation.js +5 -0
- package/lib/module/native/NativeImageSegmentation.js.map +1 -0
- package/lib/module/native/NativeLLM.js.map +1 -1
- package/lib/module/native/NativeTextEmbeddings.js +5 -0
- package/lib/module/native/NativeTextEmbeddings.js.map +1 -0
- package/lib/module/native/NativeTokenizer.js +5 -0
- package/lib/module/native/NativeTokenizer.js.map +1 -0
- package/lib/module/native/RnExecutorchModules.js +18 -113
- package/lib/module/native/RnExecutorchModules.js.map +1 -1
- package/lib/module/types/common.js.map +1 -1
- package/lib/module/types/imageSegmentation.js +29 -0
- package/lib/module/types/imageSegmentation.js.map +1 -0
- package/lib/module/types/llm.js +7 -0
- package/lib/module/types/llm.js.map +1 -0
- package/lib/module/types/{object_detection.js → objectDetection.js} +1 -1
- package/lib/module/types/objectDetection.js.map +1 -0
- package/lib/module/types/ocr.js +2 -0
- package/lib/module/types/stt.js +82 -0
- package/lib/module/types/stt.js.map +1 -0
- package/lib/module/utils/ResourceFetcher.js +156 -0
- package/lib/module/utils/ResourceFetcher.js.map +1 -0
- package/lib/module/utils/llm.js +25 -0
- package/lib/module/utils/llm.js.map +1 -0
- package/lib/module/utils/stt.js +22 -0
- package/lib/module/utils/stt.js.map +1 -0
- package/lib/typescript/Error.d.ts +4 -1
- package/lib/typescript/Error.d.ts.map +1 -1
- package/lib/typescript/constants/{llamaDefaults.d.ts → llmDefaults.d.ts} +5 -5
- package/lib/typescript/constants/llmDefaults.d.ts.map +1 -0
- package/lib/typescript/constants/modelUrls.d.ts +74 -28
- package/lib/typescript/constants/modelUrls.d.ts.map +1 -1
- package/lib/typescript/constants/ocr/models.d.ts +285 -0
- package/lib/typescript/constants/ocr/models.d.ts.map +1 -0
- package/lib/typescript/constants/ocr/symbols.d.ts +73 -1
- package/lib/typescript/constants/ocr/symbols.d.ts.map +1 -1
- package/lib/typescript/constants/sttDefaults.d.ts +8 -13
- package/lib/typescript/constants/sttDefaults.d.ts.map +1 -1
- package/lib/typescript/controllers/LLMController.d.ts +46 -0
- package/lib/typescript/controllers/LLMController.d.ts.map +1 -0
- package/lib/typescript/controllers/OCRController.d.ts.map +1 -1
- package/lib/typescript/controllers/SpeechToTextController.d.ts +30 -16
- package/lib/typescript/controllers/SpeechToTextController.d.ts.map +1 -1
- package/lib/typescript/controllers/VerticalOCRController.d.ts +1 -1
- package/lib/typescript/controllers/VerticalOCRController.d.ts.map +1 -1
- package/lib/typescript/hooks/computer_vision/useClassification.d.ts +5 -5
- package/lib/typescript/hooks/computer_vision/useClassification.d.ts.map +1 -1
- package/lib/typescript/hooks/computer_vision/useImageSegmentation.d.ts +37 -0
- package/lib/typescript/hooks/computer_vision/useImageSegmentation.d.ts.map +1 -0
- package/lib/typescript/hooks/computer_vision/useOCR.d.ts +2 -1
- package/lib/typescript/hooks/computer_vision/useOCR.d.ts.map +1 -1
- package/lib/typescript/hooks/computer_vision/useObjectDetection.d.ts +5 -4
- package/lib/typescript/hooks/computer_vision/useObjectDetection.d.ts.map +1 -1
- package/lib/typescript/hooks/computer_vision/useStyleTransfer.d.ts +4 -2
- package/lib/typescript/hooks/computer_vision/useStyleTransfer.d.ts.map +1 -1
- package/lib/typescript/hooks/computer_vision/useVerticalOCR.d.ts +2 -1
- package/lib/typescript/hooks/computer_vision/useVerticalOCR.d.ts.map +1 -1
- package/lib/typescript/hooks/general/useExecutorchModule.d.ts +5 -6
- package/lib/typescript/hooks/general/useExecutorchModule.d.ts.map +1 -1
- package/lib/typescript/hooks/natural_language_processing/useLLM.d.ts +6 -6
- package/lib/typescript/hooks/natural_language_processing/useLLM.d.ts.map +1 -1
- package/lib/typescript/hooks/natural_language_processing/useSpeechToText.d.ts +7 -3
- package/lib/typescript/hooks/natural_language_processing/useSpeechToText.d.ts.map +1 -1
- package/lib/typescript/hooks/natural_language_processing/useTextEmbeddings.d.ts +13 -0
- package/lib/typescript/hooks/natural_language_processing/useTextEmbeddings.d.ts.map +1 -0
- package/lib/typescript/hooks/natural_language_processing/useTokenizer.d.ts +16 -0
- package/lib/typescript/hooks/natural_language_processing/useTokenizer.d.ts.map +1 -0
- package/lib/typescript/hooks/useModule.d.ts +11 -10
- package/lib/typescript/hooks/useModule.d.ts.map +1 -1
- package/lib/typescript/index.d.ts +15 -2
- package/lib/typescript/index.d.ts.map +1 -1
- package/lib/typescript/modules/BaseModule.d.ts +4 -5
- package/lib/typescript/modules/BaseModule.d.ts.map +1 -1
- package/lib/typescript/modules/computer_vision/ClassificationModule.d.ts +7 -7
- package/lib/typescript/modules/computer_vision/ClassificationModule.d.ts.map +1 -1
- package/lib/typescript/modules/computer_vision/ImageSegmentationModule.d.ts +32 -0
- package/lib/typescript/modules/computer_vision/ImageSegmentationModule.d.ts.map +1 -0
- package/lib/typescript/modules/computer_vision/ObjectDetectionModule.d.ts +6 -5
- package/lib/typescript/modules/computer_vision/ObjectDetectionModule.d.ts.map +1 -1
- package/lib/typescript/modules/computer_vision/StyleTransferModule.d.ts +6 -5
- package/lib/typescript/modules/computer_vision/StyleTransferModule.d.ts.map +1 -1
- package/lib/typescript/modules/general/ExecutorchModule.d.ts +4 -3
- package/lib/typescript/modules/general/ExecutorchModule.d.ts.map +1 -1
- package/lib/typescript/modules/natural_language_processing/LLMModule.d.ts +19 -5
- package/lib/typescript/modules/natural_language_processing/LLMModule.d.ts.map +1 -1
- package/lib/typescript/modules/natural_language_processing/SpeechToTextModule.d.ts +7 -4
- package/lib/typescript/modules/natural_language_processing/SpeechToTextModule.d.ts.map +1 -1
- package/lib/typescript/modules/natural_language_processing/TextEmbeddingsModule.d.ts +8 -0
- package/lib/typescript/modules/natural_language_processing/TextEmbeddingsModule.d.ts.map +1 -0
- package/lib/typescript/modules/natural_language_processing/TokenizerModule.d.ts +12 -0
- package/lib/typescript/modules/natural_language_processing/TokenizerModule.d.ts.map +1 -0
- package/lib/typescript/native/NativeClassification.d.ts.map +1 -1
- package/lib/typescript/native/NativeImageSegmentation.d.ts +10 -0
- package/lib/typescript/native/NativeImageSegmentation.d.ts.map +1 -0
- package/lib/typescript/native/NativeLLM.d.ts +3 -4
- package/lib/typescript/native/NativeLLM.d.ts.map +1 -1
- package/lib/typescript/native/NativeObjectDetection.d.ts +1 -1
- package/lib/typescript/native/NativeObjectDetection.d.ts.map +1 -1
- package/lib/typescript/native/NativeSpeechToText.d.ts +2 -2
- package/lib/typescript/native/NativeSpeechToText.d.ts.map +1 -1
- package/lib/typescript/native/NativeTextEmbeddings.d.ts +8 -0
- package/lib/typescript/native/NativeTextEmbeddings.d.ts.map +1 -0
- package/lib/typescript/native/NativeTokenizer.d.ts +12 -0
- package/lib/typescript/native/NativeTokenizer.d.ts.map +1 -0
- package/lib/typescript/native/RnExecutorchModules.d.ts +18 -41
- package/lib/typescript/native/RnExecutorchModules.d.ts.map +1 -1
- package/lib/typescript/types/common.d.ts +1 -26
- package/lib/typescript/types/common.d.ts.map +1 -1
- package/lib/typescript/types/imageSegmentation.d.ts +25 -0
- package/lib/typescript/types/imageSegmentation.d.ts.map +1 -0
- package/lib/typescript/types/llm.d.ts +38 -0
- package/lib/typescript/types/llm.d.ts.map +1 -0
- package/lib/typescript/types/{object_detection.d.ts → objectDetection.d.ts} +1 -1
- package/lib/typescript/types/objectDetection.d.ts.map +1 -0
- package/lib/typescript/types/ocr.d.ts +2 -1
- package/lib/typescript/types/ocr.d.ts.map +1 -1
- package/lib/typescript/types/stt.d.ts +91 -0
- package/lib/typescript/types/stt.d.ts.map +1 -0
- package/lib/typescript/utils/ResourceFetcher.d.ts +17 -0
- package/lib/typescript/utils/ResourceFetcher.d.ts.map +1 -0
- package/lib/typescript/utils/llm.d.ts +3 -0
- package/lib/typescript/utils/llm.d.ts.map +1 -0
- package/lib/typescript/utils/stt.d.ts +2 -0
- package/lib/typescript/utils/stt.d.ts.map +1 -0
- package/package.json +13 -49
- package/react-native-executorch.podspec +1 -1
- package/src/Error.ts +16 -3
- package/src/constants/llmDefaults.ts +14 -0
- package/src/constants/modelUrls.ts +146 -39
- package/src/constants/ocr/models.ts +453 -0
- package/src/constants/ocr/symbols.ts +147 -3
- package/src/constants/sttDefaults.ts +55 -37
- package/src/controllers/LLMController.ts +286 -0
- package/src/controllers/OCRController.ts +14 -28
- package/src/controllers/SpeechToTextController.ts +318 -180
- package/src/controllers/VerticalOCRController.ts +17 -32
- package/src/hooks/computer_vision/useClassification.ts +11 -26
- package/src/hooks/computer_vision/useImageSegmentation.ts +18 -0
- package/src/hooks/computer_vision/useOCR.ts +17 -5
- package/src/hooks/computer_vision/useObjectDetection.ts +10 -24
- package/src/hooks/computer_vision/useStyleTransfer.ts +9 -25
- package/src/hooks/computer_vision/useVerticalOCR.ts +11 -4
- package/src/hooks/general/useExecutorchModule.ts +10 -50
- package/src/hooks/natural_language_processing/useLLM.ts +80 -97
- package/src/hooks/natural_language_processing/useSpeechToText.ts +39 -12
- package/src/hooks/natural_language_processing/useTextEmbeddings.ts +18 -0
- package/src/hooks/natural_language_processing/useTokenizer.ts +61 -0
- package/src/hooks/useModule.ts +32 -92
- package/src/index.tsx +16 -2
- package/src/modules/BaseModule.ts +16 -26
- package/src/modules/computer_vision/ClassificationModule.ts +13 -8
- package/src/modules/computer_vision/ImageSegmentationModule.ts +39 -0
- package/src/modules/computer_vision/ObjectDetectionModule.ts +13 -8
- package/src/modules/computer_vision/StyleTransferModule.ts +13 -8
- package/src/modules/general/ExecutorchModule.ts +11 -6
- package/src/modules/natural_language_processing/LLMModule.ts +64 -51
- package/src/modules/natural_language_processing/SpeechToTextModule.ts +25 -10
- package/src/modules/natural_language_processing/TextEmbeddingsModule.ts +18 -0
- package/src/modules/natural_language_processing/TokenizerModule.ts +34 -0
- package/src/native/NativeClassification.ts +0 -1
- package/src/native/NativeImageSegmentation.ts +14 -0
- package/src/native/NativeLLM.ts +3 -10
- package/src/native/NativeObjectDetection.ts +1 -1
- package/src/native/NativeSpeechToText.ts +2 -2
- package/src/native/NativeTextEmbeddings.ts +9 -0
- package/src/native/NativeTokenizer.ts +13 -0
- package/src/native/RnExecutorchModules.ts +54 -234
- package/src/types/common.ts +1 -44
- package/src/types/imageSegmentation.ts +25 -0
- package/src/types/llm.ts +57 -0
- package/src/types/ocr.ts +3 -1
- package/src/types/stt.ts +93 -0
- package/src/utils/ResourceFetcher.ts +196 -0
- package/src/utils/llm.ts +34 -0
- package/src/utils/stt.ts +28 -0
- package/android/src/main/java/com/swmansion/rnexecutorch/utils/llms/ConversationManager.kt +0 -68
- package/ios/ExecutorchLib.xcframework/ios-arm64-simulator/ExecutorchLib.framework/_CodeSignature/CodeResources +0 -124
- package/ios/RnExecutorch/utils/llms/Constants.h +0 -6
- package/ios/RnExecutorch/utils/llms/Constants.mm +0 -23
- package/ios/RnExecutorch/utils/llms/ConversationManager.h +0 -26
- package/ios/RnExecutorch/utils/llms/ConversationManager.mm +0 -71
- package/lib/module/constants/llamaDefaults.js.map +0 -1
- package/lib/module/modules/computer_vision/BaseCVModule.js +0 -14
- package/lib/module/modules/computer_vision/BaseCVModule.js.map +0 -1
- package/lib/module/types/object_detection.js.map +0 -1
- package/lib/module/utils/fetchResource.js +0 -93
- package/lib/module/utils/fetchResource.js.map +0 -1
- package/lib/module/utils/listDownloadedResources.js +0 -13
- package/lib/module/utils/listDownloadedResources.js.map +0 -1
- package/lib/typescript/constants/llamaDefaults.d.ts.map +0 -1
- package/lib/typescript/modules/computer_vision/BaseCVModule.d.ts +0 -9
- package/lib/typescript/modules/computer_vision/BaseCVModule.d.ts.map +0 -1
- package/lib/typescript/types/object_detection.d.ts.map +0 -1
- package/lib/typescript/utils/fetchResource.d.ts +0 -3
- package/lib/typescript/utils/fetchResource.d.ts.map +0 -1
- package/lib/typescript/utils/listDownloadedResources.d.ts +0 -3
- package/lib/typescript/utils/listDownloadedResources.d.ts.map +0 -1
- package/src/constants/llamaDefaults.ts +0 -9
- package/src/modules/computer_vision/BaseCVModule.ts +0 -22
- package/src/utils/fetchResource.ts +0 -106
- package/src/utils/listDownloadedResources.ts +0 -12
- /package/src/types/{object_detection.ts → objectDetection.ts} +0 -0
|
@@ -0,0 +1,146 @@
|
|
|
1
|
+
#import "ImageSegmentationModel.h"
|
|
2
|
+
#import "../../utils/Conversions.h"
|
|
3
|
+
#import "../../utils/ImageProcessor.h"
|
|
4
|
+
#import "../../utils/Numerical.h"
|
|
5
|
+
#import "Constants.h"
|
|
6
|
+
#import <unordered_set>
|
|
7
|
+
|
|
8
|
+
@interface ImageSegmentationModel ()
|
|
9
|
+
- (NSArray *)preprocess:(cv::Mat &)input;
|
|
10
|
+
- (NSDictionary *)postprocess:(NSArray *)output
|
|
11
|
+
returnClasses:(NSArray *)classesOfInterest
|
|
12
|
+
resize:(BOOL)resize;
|
|
13
|
+
@end
|
|
14
|
+
|
|
15
|
+
@implementation ImageSegmentationModel {
|
|
16
|
+
cv::Size originalSize;
|
|
17
|
+
}
|
|
18
|
+
|
|
19
|
+
- (cv::Size)getModelImageSize {
|
|
20
|
+
NSArray *inputShape = [module getInputShape:@0];
|
|
21
|
+
NSNumber *widthNumber = inputShape.lastObject;
|
|
22
|
+
NSNumber *heightNumber = inputShape[inputShape.count - 2];
|
|
23
|
+
|
|
24
|
+
int height = [heightNumber intValue];
|
|
25
|
+
int width = [widthNumber intValue];
|
|
26
|
+
|
|
27
|
+
return cv::Size(height, width);
|
|
28
|
+
}
|
|
29
|
+
|
|
30
|
+
- (NSArray *)preprocess:(cv::Mat &)input {
|
|
31
|
+
originalSize = cv::Size(input.cols, input.rows);
|
|
32
|
+
|
|
33
|
+
cv::Size modelImageSize = [self getModelImageSize];
|
|
34
|
+
cv::Mat output;
|
|
35
|
+
cv::resize(input, output, modelImageSize);
|
|
36
|
+
|
|
37
|
+
NSArray *modelInput = [ImageProcessor matToNSArray:output];
|
|
38
|
+
return modelInput;
|
|
39
|
+
}
|
|
40
|
+
|
|
41
|
+
std::vector<cv::Mat> extractResults(NSArray *result, std::size_t numLabels,
|
|
42
|
+
cv::Size modelImageSize,
|
|
43
|
+
cv::Size originalSize, BOOL resize) {
|
|
44
|
+
std::size_t numModelPixels = modelImageSize.height * modelImageSize.width;
|
|
45
|
+
|
|
46
|
+
std::vector<cv::Mat> resizedLabelScores(numLabels);
|
|
47
|
+
for (std::size_t label = 0; label < numLabels; ++label) {
|
|
48
|
+
cv::Mat labelMat = cv::Mat(modelImageSize, CV_64F);
|
|
49
|
+
|
|
50
|
+
for (std::size_t pixel = 0; pixel < numModelPixels; ++pixel) {
|
|
51
|
+
int row = pixel / modelImageSize.width;
|
|
52
|
+
int col = pixel % modelImageSize.width;
|
|
53
|
+
labelMat.at<double>(row, col) =
|
|
54
|
+
[result[label * numModelPixels + pixel] doubleValue];
|
|
55
|
+
}
|
|
56
|
+
|
|
57
|
+
if (resize) {
|
|
58
|
+
cv::resize(labelMat, resizedLabelScores[label], originalSize);
|
|
59
|
+
} else {
|
|
60
|
+
resizedLabelScores[label] = std::move(labelMat);
|
|
61
|
+
}
|
|
62
|
+
}
|
|
63
|
+
return resizedLabelScores;
|
|
64
|
+
}
|
|
65
|
+
|
|
66
|
+
void adjustScoresPerPixel(std::vector<cv::Mat> &labelScores, cv::Mat &argMax,
|
|
67
|
+
cv::Size outputSize, std::size_t numLabels) {
|
|
68
|
+
std::size_t numOutputPixels = outputSize.height * outputSize.width;
|
|
69
|
+
for (std::size_t pixel = 0; pixel < numOutputPixels; ++pixel) {
|
|
70
|
+
int row = pixel / outputSize.width;
|
|
71
|
+
int col = pixel % outputSize.width;
|
|
72
|
+
std::vector<double> scores;
|
|
73
|
+
scores.reserve(numLabels);
|
|
74
|
+
for (const auto &mat : labelScores) {
|
|
75
|
+
scores.push_back(mat.at<double>(row, col));
|
|
76
|
+
}
|
|
77
|
+
|
|
78
|
+
std::vector<double> adjustedScores = softmax(scores);
|
|
79
|
+
|
|
80
|
+
for (std::size_t label = 0; label < numLabels; ++label) {
|
|
81
|
+
labelScores[label].at<double>(row, col) = adjustedScores[label];
|
|
82
|
+
}
|
|
83
|
+
|
|
84
|
+
auto maxIt = std::max_element(scores.begin(), scores.end());
|
|
85
|
+
argMax.at<int>(row, col) = std::distance(scores.begin(), maxIt);
|
|
86
|
+
}
|
|
87
|
+
}
|
|
88
|
+
|
|
89
|
+
- (NSDictionary *)postprocess:(NSArray *)output
|
|
90
|
+
returnClasses:(NSArray *)classesOfInterest
|
|
91
|
+
resize:(BOOL)resize {
|
|
92
|
+
cv::Size modelImageSize = [self getModelImageSize];
|
|
93
|
+
|
|
94
|
+
std::size_t numLabels = deeplabv3_resnet50_labels.size();
|
|
95
|
+
|
|
96
|
+
NSAssert((std::size_t)output.count ==
|
|
97
|
+
numLabels * modelImageSize.height * modelImageSize.width,
|
|
98
|
+
@"Model generated unexpected output size.");
|
|
99
|
+
|
|
100
|
+
// For each label extract it's matrix,
|
|
101
|
+
// and rescale it to the original size if `resize`
|
|
102
|
+
std::vector<cv::Mat> resizedLabelScores =
|
|
103
|
+
extractResults(output, numLabels, modelImageSize, originalSize, resize);
|
|
104
|
+
|
|
105
|
+
cv::Size outputSize = resize ? originalSize : modelImageSize;
|
|
106
|
+
cv::Mat argMax = cv::Mat(outputSize, CV_32S);
|
|
107
|
+
|
|
108
|
+
// For each pixel apply softmax across all the labels and calculate the argMax
|
|
109
|
+
adjustScoresPerPixel(resizedLabelScores, argMax, outputSize, numLabels);
|
|
110
|
+
|
|
111
|
+
std::unordered_set<std::string> labelSet;
|
|
112
|
+
|
|
113
|
+
for (id label in classesOfInterest) {
|
|
114
|
+
labelSet.insert(std::string([label UTF8String]));
|
|
115
|
+
}
|
|
116
|
+
|
|
117
|
+
NSMutableDictionary *result = [NSMutableDictionary dictionary];
|
|
118
|
+
|
|
119
|
+
// Convert to NSArray and populate the final dictionary
|
|
120
|
+
for (std::size_t label = 0; label < numLabels; ++label) {
|
|
121
|
+
if (labelSet.contains(deeplabv3_resnet50_labels[label])) {
|
|
122
|
+
NSString *labelString = @(deeplabv3_resnet50_labels[label].c_str());
|
|
123
|
+
NSArray *arr = simpleMatToNSArray<double>(resizedLabelScores[label]);
|
|
124
|
+
result[labelString] = arr;
|
|
125
|
+
}
|
|
126
|
+
}
|
|
127
|
+
|
|
128
|
+
result[@"ARGMAX"] = simpleMatToNSArray<int>(argMax);
|
|
129
|
+
|
|
130
|
+
return result;
|
|
131
|
+
}
|
|
132
|
+
|
|
133
|
+
- (NSDictionary *)runModel:(cv::Mat &)input
|
|
134
|
+
returnClasses:(NSArray *)classesOfInterest
|
|
135
|
+
resize:(BOOL)resize {
|
|
136
|
+
NSArray *modelInput = [self preprocess:input];
|
|
137
|
+
NSArray *result = [self forward:@[ modelInput ]];
|
|
138
|
+
|
|
139
|
+
NSDictionary *output = [self postprocess:result[0]
|
|
140
|
+
returnClasses:classesOfInterest
|
|
141
|
+
resize:resize];
|
|
142
|
+
|
|
143
|
+
return output;
|
|
144
|
+
}
|
|
145
|
+
|
|
146
|
+
@end
|
|
@@ -1,7 +1,6 @@
|
|
|
1
1
|
#include "SSDLiteLargeModel.hpp"
|
|
2
2
|
#include "../../utils/ObjectDetectionUtils.hpp"
|
|
3
3
|
#include "ImageProcessor.h"
|
|
4
|
-
#include <vector>
|
|
5
4
|
|
|
6
5
|
float constexpr iouThreshold = 0.55;
|
|
7
6
|
float constexpr detectionThreshold = 0.7;
|
|
@@ -54,7 +53,7 @@ int constexpr modelInputHeight = 320;
|
|
|
54
53
|
int inputImageWidth = size.width;
|
|
55
54
|
int inputImageHeight = size.height;
|
|
56
55
|
NSArray *modelInput = [self preprocess:input];
|
|
57
|
-
NSArray *forwardResult = [self forward
|
|
56
|
+
NSArray *forwardResult = [self forward:@[ modelInput ]];
|
|
58
57
|
NSArray *output =
|
|
59
58
|
[self postprocess:forwardResult
|
|
60
59
|
widthRatio:inputImageWidth / (float)modelInputWidth
|
|
@@ -1,5 +1,6 @@
|
|
|
1
1
|
#import "Detector.h"
|
|
2
2
|
#import "../../utils/ImageProcessor.h"
|
|
3
|
+
#import "utils/Constants.h"
|
|
3
4
|
#import "utils/DetectorUtils.h"
|
|
4
5
|
#import "utils/OCRUtils.h"
|
|
5
6
|
|
|
@@ -92,7 +93,7 @@
|
|
|
92
93
|
|
|
93
94
|
- (NSArray *)runModel:(cv::Mat &)input {
|
|
94
95
|
NSArray *modelInput = [self preprocess:input];
|
|
95
|
-
NSArray *modelResult = [self forward
|
|
96
|
+
NSArray *modelResult = [self forward:@[ modelInput ]];
|
|
96
97
|
NSArray *result = [self postprocess:modelResult];
|
|
97
98
|
return result;
|
|
98
99
|
}
|
|
@@ -3,10 +3,11 @@
|
|
|
3
3
|
@interface RecognitionHandler : NSObject
|
|
4
4
|
|
|
5
5
|
- (instancetype)initWithSymbols:(NSString *)symbols;
|
|
6
|
-
|
|
7
|
-
|
|
8
|
-
|
|
9
|
-
|
|
6
|
+
|
|
7
|
+
- (NSNumber *)loadRecognizers:(NSString *)largeRecognizerPath
|
|
8
|
+
mediumRecognizerPath:(NSString *)mediumRecognizerPath
|
|
9
|
+
smallRecognizerPath:(NSString *)smallRecognizerPath;
|
|
10
|
+
|
|
10
11
|
- (NSArray *)recognize:(NSArray<NSDictionary *> *)bBoxesList
|
|
11
12
|
imgGray:(cv::Mat)imgGray
|
|
12
13
|
desiredWidth:(int)desiredWidth
|
|
@@ -1,12 +1,9 @@
|
|
|
1
1
|
#import "RecognitionHandler.h"
|
|
2
|
-
#import "../../utils/ImageProcessor.h"
|
|
3
2
|
#import "./utils/CTCLabelConverter.h"
|
|
4
3
|
#import "./utils/Constants.h"
|
|
5
4
|
#import "./utils/OCRUtils.h"
|
|
6
5
|
#import "./utils/RecognizerUtils.h"
|
|
7
|
-
#import "ExecutorchLib/ETModel.h"
|
|
8
6
|
#import "Recognizer.h"
|
|
9
|
-
#import <React/RCTBridgeModule.h>
|
|
10
7
|
|
|
11
8
|
/*
|
|
12
9
|
RecognitionHandler class is responsible for loading and choosing the
|
|
@@ -34,15 +31,12 @@
|
|
|
34
31
|
return self;
|
|
35
32
|
}
|
|
36
33
|
|
|
37
|
-
- (
|
|
38
|
-
|
|
39
|
-
|
|
40
|
-
completion:(void (^)(BOOL, NSNumber *))completion {
|
|
41
|
-
dispatch_group_t group = dispatch_group_create();
|
|
42
|
-
__block BOOL allSuccessful = YES;
|
|
43
|
-
|
|
34
|
+
- (NSNumber *)loadRecognizers:(NSString *)largeRecognizerPath
|
|
35
|
+
mediumRecognizerPath:(NSString *)mediumRecognizerPath
|
|
36
|
+
smallRecognizerPath:(NSString *)smallRecognizerPath {
|
|
44
37
|
NSArray<Recognizer *> *recognizers =
|
|
45
38
|
@[ recognizerLarge, recognizerMedium, recognizerSmall ];
|
|
39
|
+
|
|
46
40
|
NSArray<NSString *> *paths =
|
|
47
41
|
@[ largeRecognizerPath, mediumRecognizerPath, smallRecognizerPath ];
|
|
48
42
|
|
|
@@ -50,24 +44,13 @@
|
|
|
50
44
|
Recognizer *recognizer = recognizers[i];
|
|
51
45
|
NSString *path = paths[i];
|
|
52
46
|
|
|
53
|
-
|
|
54
|
-
|
|
55
|
-
|
|
56
|
-
|
|
57
|
-
allSuccessful = NO;
|
|
58
|
-
dispatch_group_leave(group);
|
|
59
|
-
completion(NO, errorCode);
|
|
60
|
-
return;
|
|
61
|
-
}
|
|
62
|
-
dispatch_group_leave(group);
|
|
63
|
-
}];
|
|
47
|
+
NSNumber *errorCode = [recognizer loadModel:path];
|
|
48
|
+
if ([errorCode intValue] != 0) {
|
|
49
|
+
return errorCode;
|
|
50
|
+
}
|
|
64
51
|
}
|
|
65
52
|
|
|
66
|
-
|
|
67
|
-
if (allSuccessful) {
|
|
68
|
-
completion(YES, @(0));
|
|
69
|
-
}
|
|
70
|
-
});
|
|
53
|
+
return @0;
|
|
71
54
|
}
|
|
72
55
|
|
|
73
56
|
- (NSArray *)runModel:(cv::Mat)croppedImage {
|
|
@@ -1,7 +1,6 @@
|
|
|
1
1
|
#import "Recognizer.h"
|
|
2
2
|
#import "../../utils/ImageProcessor.h"
|
|
3
3
|
#import "RecognizerUtils.h"
|
|
4
|
-
#import "utils/OCRUtils.h"
|
|
5
4
|
|
|
6
5
|
/*
|
|
7
6
|
The model used as detector is based on CRNN paper.
|
|
@@ -69,7 +68,7 @@
|
|
|
69
68
|
|
|
70
69
|
- (NSArray *)runModel:(cv::Mat &)input {
|
|
71
70
|
NSArray *modelInput = [self preprocess:input];
|
|
72
|
-
NSArray *modelResult = [self forward
|
|
71
|
+
NSArray *modelResult = [self forward:@[ modelInput ]];
|
|
73
72
|
NSArray *result = [self postprocess:modelResult];
|
|
74
73
|
|
|
75
74
|
return result;
|
|
@@ -1,5 +1,6 @@
|
|
|
1
1
|
#import "VerticalDetector.h"
|
|
2
2
|
#import "../../utils/ImageProcessor.h"
|
|
3
|
+
#import "utils/Constants.h"
|
|
3
4
|
#import "utils/DetectorUtils.h"
|
|
4
5
|
#import "utils/OCRUtils.h"
|
|
5
6
|
|
|
@@ -109,7 +110,7 @@
|
|
|
109
110
|
|
|
110
111
|
- (NSArray *)runModel:(cv::Mat &)input {
|
|
111
112
|
NSArray *modelInput = [self preprocess:input];
|
|
112
|
-
NSArray *modelResult = [self forward
|
|
113
|
+
NSArray *modelResult = [self forward:@[ modelInput ]];
|
|
113
114
|
NSArray *result = [self postprocess:modelResult];
|
|
114
115
|
return result;
|
|
115
116
|
}
|
|
@@ -1,9 +1,4 @@
|
|
|
1
1
|
#import "Moonshine.hpp"
|
|
2
|
-
#import "ExecutorchLib/ETModel.h"
|
|
3
|
-
#import "MoonshineEncoder.hpp"
|
|
4
|
-
#import "Moonshinedecoder.hpp"
|
|
5
|
-
#import <Foundation/Foundation.h>
|
|
6
|
-
#import <UIKit/UIKit.h>
|
|
7
2
|
|
|
8
3
|
@implementation Moonshine {
|
|
9
4
|
MoonshineEncoder *encoder;
|
|
@@ -66,4 +61,4 @@
|
|
|
66
61
|
}];
|
|
67
62
|
}
|
|
68
63
|
|
|
69
|
-
@end
|
|
64
|
+
@end
|
|
@@ -1,6 +1,4 @@
|
|
|
1
1
|
#import "SpeechToTextBaseModel.hpp"
|
|
2
|
-
#import <Foundation/Foundation.h>
|
|
3
|
-
#import <UIKit/UIKit.h>
|
|
4
2
|
|
|
5
3
|
@implementation SpeechToTextBaseModel
|
|
6
4
|
|
|
@@ -8,16 +6,14 @@
|
|
|
8
6
|
withSource:(NSString *)source
|
|
9
7
|
onSuccess:(void (^)(void))success
|
|
10
8
|
onFailure:(void (^)(NSString *))failure {
|
|
9
|
+
NSNumber *errorCode = [model loadModel:source];
|
|
11
10
|
|
|
12
|
-
|
|
13
|
-
|
|
14
|
-
|
|
15
|
-
|
|
16
|
-
|
|
17
|
-
|
|
18
|
-
stringWithFormat:@"%ld", (long)[errorCode longValue]]);
|
|
19
|
-
}
|
|
20
|
-
}];
|
|
11
|
+
if ([errorCode intValue] != 0) {
|
|
12
|
+
failure([NSString stringWithFormat:@"%ld", (long)[errorCode longValue]]);
|
|
13
|
+
return;
|
|
14
|
+
}
|
|
15
|
+
|
|
16
|
+
success();
|
|
21
17
|
}
|
|
22
18
|
|
|
23
19
|
@end
|
|
@@ -1,11 +1,6 @@
|
|
|
1
1
|
#import "Whisper.hpp"
|
|
2
|
-
#import "../../utils/SFFT.hpp"
|
|
3
|
-
#import "../../utils/ScalarType.h"
|
|
4
|
-
#import "ExecutorchLib/ETModel.h"
|
|
5
2
|
#import "WhisperEncoder.hpp"
|
|
6
3
|
#import "Whisperdecoder.hpp"
|
|
7
|
-
#import <Foundation/Foundation.h>
|
|
8
|
-
#import <UIKit/UIKit.h>
|
|
9
4
|
|
|
10
5
|
@implementation Whisper {
|
|
11
6
|
WhisperEncoder *encoder;
|
package/ios/RnExecutorch/models/{StyleTransferModel.mm → style_transfer/StyleTransferModel.mm}
RENAMED
|
@@ -1,6 +1,5 @@
|
|
|
1
1
|
#import "StyleTransferModel.h"
|
|
2
|
-
#import "
|
|
3
|
-
#import "opencv2/opencv.hpp"
|
|
2
|
+
#import "../../utils/ImageProcessor.h"
|
|
4
3
|
|
|
5
4
|
@implementation StyleTransferModel {
|
|
6
5
|
cv::Size originalSize;
|
|
@@ -42,7 +41,7 @@
|
|
|
42
41
|
|
|
43
42
|
- (cv::Mat)runModel:(cv::Mat &)input {
|
|
44
43
|
NSArray *modelInput = [self preprocess:input];
|
|
45
|
-
NSArray *result = [self forward
|
|
44
|
+
NSArray *result = [self forward:@[ modelInput ]];
|
|
46
45
|
input = [self postprocess:result[0]];
|
|
47
46
|
|
|
48
47
|
return input;
|
|
@@ -0,0 +1,15 @@
|
|
|
1
|
+
#import "BaseModel.h"
|
|
2
|
+
#import "ExecutorchLib/HuggingFaceTokenizer.h"
|
|
3
|
+
|
|
4
|
+
@interface TextEmbeddingsModel : BaseModel {
|
|
5
|
+
@protected
|
|
6
|
+
HuggingFaceTokenizer *tokenizer;
|
|
7
|
+
}
|
|
8
|
+
|
|
9
|
+
- (void)loadTokenizer:(NSString *)tokenizerSource;
|
|
10
|
+
- (NSArray *)preprocess:(NSString *)input;
|
|
11
|
+
- (NSArray *)runModel:(NSString *)input;
|
|
12
|
+
- (NSArray *)postprocess:(NSArray *)input
|
|
13
|
+
attentionMask:(NSArray *)attentionMask;
|
|
14
|
+
|
|
15
|
+
@end
|
|
@@ -0,0 +1,45 @@
|
|
|
1
|
+
#import "TextEmbeddingsModel.h"
|
|
2
|
+
#import "TextEmbeddingsUtils.h"
|
|
3
|
+
|
|
4
|
+
@implementation TextEmbeddingsModel
|
|
5
|
+
|
|
6
|
+
- (NSArray *)preprocess:(NSString *)input {
|
|
7
|
+
NSArray *input_ids = [self->tokenizer encode:input];
|
|
8
|
+
NSMutableArray *attention_mask = [NSMutableArray new];
|
|
9
|
+
for (int i = 0; i < [input_ids count]; i++) {
|
|
10
|
+
[attention_mask addObject:@((int)([input_ids[i] intValue] != 0))];
|
|
11
|
+
}
|
|
12
|
+
return @[ input_ids, attention_mask ]; // [2, tokens]
|
|
13
|
+
}
|
|
14
|
+
|
|
15
|
+
- (NSArray *)postprocess:(NSArray *)modelOutput // [tokens * embedding_dim]
|
|
16
|
+
attentionMask:(NSArray *)attentionMask // [tokens]
|
|
17
|
+
{
|
|
18
|
+
NSArray *embeddings = [TextEmbeddingsUtils meanPooling:modelOutput
|
|
19
|
+
attentionMask:attentionMask];
|
|
20
|
+
return [TextEmbeddingsUtils normalize:embeddings];
|
|
21
|
+
}
|
|
22
|
+
|
|
23
|
+
- (NSArray *)runModel:(NSString *)input {
|
|
24
|
+
NSArray *modelInput = [self preprocess:input];
|
|
25
|
+
|
|
26
|
+
NSMutableArray *inputTypes = [NSMutableArray arrayWithObjects:@4, @4, nil];
|
|
27
|
+
NSMutableArray *shapes = [NSMutableArray new];
|
|
28
|
+
|
|
29
|
+
NSNumber *tokenCount = @([modelInput[0] count]);
|
|
30
|
+
for (__unused id _ in modelInput) {
|
|
31
|
+
[shapes addObject:[NSMutableArray arrayWithObjects:@1, tokenCount, nil]];
|
|
32
|
+
}
|
|
33
|
+
|
|
34
|
+
NSArray *modelOutput = [self forward:modelInput
|
|
35
|
+
shapes:shapes
|
|
36
|
+
inputTypes:inputTypes];
|
|
37
|
+
return [self postprocess:modelOutput[0] attentionMask:modelInput[1]];
|
|
38
|
+
}
|
|
39
|
+
|
|
40
|
+
- (void)loadTokenizer:(NSString *)tokenizerSource {
|
|
41
|
+
tokenizer =
|
|
42
|
+
[[HuggingFaceTokenizer alloc] initWithTokenizerPath:tokenizerSource];
|
|
43
|
+
}
|
|
44
|
+
|
|
45
|
+
@end
|
|
@@ -0,0 +1,49 @@
|
|
|
1
|
+
#import "TextEmbeddingsUtils.h"
|
|
2
|
+
|
|
3
|
+
@implementation TextEmbeddingsUtils
|
|
4
|
+
|
|
5
|
+
+ (NSArray *)meanPooling:(NSArray *)modelOutput
|
|
6
|
+
attentionMask:(NSArray *)attentionMask {
|
|
7
|
+
NSInteger modelOutputLength = [modelOutput count];
|
|
8
|
+
NSInteger attentionMaskLength = [attentionMask count];
|
|
9
|
+
NSInteger embeddingDim = modelOutputLength / attentionMaskLength;
|
|
10
|
+
|
|
11
|
+
NSMutableArray *result = [NSMutableArray arrayWithCapacity:embeddingDim];
|
|
12
|
+
|
|
13
|
+
double sumMask = 0.0;
|
|
14
|
+
for (NSNumber *maskValue in attentionMask) {
|
|
15
|
+
sumMask += [maskValue intValue];
|
|
16
|
+
}
|
|
17
|
+
sumMask = fmax(sumMask, 1e-9);
|
|
18
|
+
|
|
19
|
+
for (NSInteger i = 0; i < embeddingDim; i++) {
|
|
20
|
+
double sum = 0;
|
|
21
|
+
for (NSInteger j = 0; j < attentionMaskLength; j++) {
|
|
22
|
+
sum += [modelOutput[j * embeddingDim + i] doubleValue] *
|
|
23
|
+
[attentionMask[j] intValue];
|
|
24
|
+
}
|
|
25
|
+
[result addObject:@(sum / sumMask)];
|
|
26
|
+
}
|
|
27
|
+
|
|
28
|
+
return result;
|
|
29
|
+
}
|
|
30
|
+
|
|
31
|
+
+ (NSArray *)normalize:(NSArray *)embeddings {
|
|
32
|
+
NSInteger embeddingDim = [embeddings count];
|
|
33
|
+
double sum = 0.0;
|
|
34
|
+
|
|
35
|
+
for (NSNumber *value in embeddings) {
|
|
36
|
+
sum += [value doubleValue] * [value doubleValue];
|
|
37
|
+
}
|
|
38
|
+
sum = fmax(sqrt(sum), 1e-9);
|
|
39
|
+
|
|
40
|
+
NSMutableArray *normalizedResult =
|
|
41
|
+
[NSMutableArray arrayWithCapacity:embeddingDim];
|
|
42
|
+
for (NSNumber *value in embeddings) {
|
|
43
|
+
[normalizedResult addObject:@([value doubleValue] / sum)];
|
|
44
|
+
}
|
|
45
|
+
|
|
46
|
+
return normalizedResult;
|
|
47
|
+
}
|
|
48
|
+
|
|
49
|
+
@end
|
|
@@ -0,0 +1,15 @@
|
|
|
1
|
+
#import "opencv2/opencv.hpp"
|
|
2
|
+
|
|
3
|
+
// Convert a matrix containing a single value per cell to a NSArray
|
|
4
|
+
template <typename T>
|
|
5
|
+
NSArray* simpleMatToNSArray(const cv::Mat& mat) {
|
|
6
|
+
std::size_t numPixels = mat.rows * mat.cols;
|
|
7
|
+
NSMutableArray *arr = [[NSMutableArray alloc] initWithCapacity:numPixels];
|
|
8
|
+
|
|
9
|
+
for (std::size_t x = 0; x < mat.rows; ++x) {
|
|
10
|
+
for (std::size_t y = 0; y < mat.cols; ++y) {
|
|
11
|
+
arr[x * mat.cols + y] = @(mat.at<T>(x, y));
|
|
12
|
+
}
|
|
13
|
+
}
|
|
14
|
+
return arr;
|
|
15
|
+
}
|
package/lib/module/Error.js
CHANGED
|
@@ -1,25 +1,36 @@
|
|
|
1
1
|
"use strict";
|
|
2
2
|
|
|
3
3
|
export let ETError = /*#__PURE__*/function (ETError) {
|
|
4
|
+
// React-native-ExecuTorch errors
|
|
4
5
|
ETError[ETError["UndefinedError"] = 101] = "UndefinedError";
|
|
5
6
|
ETError[ETError["ModuleNotLoaded"] = 102] = "ModuleNotLoaded";
|
|
6
7
|
ETError[ETError["FileWriteFailed"] = 103] = "FileWriteFailed";
|
|
7
8
|
ETError[ETError["ModelGenerating"] = 104] = "ModelGenerating";
|
|
8
9
|
ETError[ETError["LanguageNotSupported"] = 105] = "LanguageNotSupported";
|
|
9
10
|
ETError[ETError["InvalidModelSource"] = 255] = "InvalidModelSource";
|
|
11
|
+
//SpeechToText errors
|
|
12
|
+
ETError[ETError["MultilingualConfiguration"] = 160] = "MultilingualConfiguration";
|
|
13
|
+
ETError[ETError["MissingDataChunk"] = 161] = "MissingDataChunk";
|
|
14
|
+
ETError[ETError["StreamingNotStarted"] = 162] = "StreamingNotStarted";
|
|
15
|
+
// ExecuTorch mapped errors
|
|
16
|
+
// Based on: https://github.com/pytorch/executorch/blob/main/runtime/core/error.h
|
|
17
|
+
// System errors
|
|
10
18
|
ETError[ETError["Ok"] = 0] = "Ok";
|
|
11
19
|
ETError[ETError["Internal"] = 1] = "Internal";
|
|
12
20
|
ETError[ETError["InvalidState"] = 2] = "InvalidState";
|
|
13
21
|
ETError[ETError["EndOfMethod"] = 3] = "EndOfMethod";
|
|
22
|
+
// Logical errors
|
|
14
23
|
ETError[ETError["NotSupported"] = 16] = "NotSupported";
|
|
15
24
|
ETError[ETError["NotImplemented"] = 17] = "NotImplemented";
|
|
16
25
|
ETError[ETError["InvalidArgument"] = 18] = "InvalidArgument";
|
|
17
26
|
ETError[ETError["InvalidType"] = 19] = "InvalidType";
|
|
18
27
|
ETError[ETError["OperatorMissing"] = 20] = "OperatorMissing";
|
|
28
|
+
// Resource errors
|
|
19
29
|
ETError[ETError["NotFound"] = 32] = "NotFound";
|
|
20
30
|
ETError[ETError["MemoryAllocationFailed"] = 33] = "MemoryAllocationFailed";
|
|
21
31
|
ETError[ETError["AccessFailed"] = 34] = "AccessFailed";
|
|
22
32
|
ETError[ETError["InvalidProgram"] = 35] = "InvalidProgram";
|
|
33
|
+
// Delegate errors
|
|
23
34
|
ETError[ETError["DelegateInvalidCompatibility"] = 48] = "DelegateInvalidCompatibility";
|
|
24
35
|
ETError[ETError["DelegateMemoryAllocationFailed"] = 49] = "DelegateMemoryAllocationFailed";
|
|
25
36
|
ETError[ETError["DelegateInvalidHandle"] = 50] = "DelegateInvalidHandle";
|
|
@@ -30,9 +41,12 @@ export const getError = e => {
|
|
|
30
41
|
if (e in ETError) return ETError[e];
|
|
31
42
|
return ETError[ETError.UndefinedError];
|
|
32
43
|
}
|
|
44
|
+
|
|
45
|
+
// try to extract number from message (can contain false positives)
|
|
33
46
|
const error = e;
|
|
34
47
|
const errorCode = parseInt(error.message, 10);
|
|
35
|
-
|
|
36
|
-
|
|
48
|
+
const message = Number.isNaN(errorCode) ? error.message : ' ' + error.message.slice(`${errorCode}`.length).trimStart();
|
|
49
|
+
const ETErrorMessage = errorCode in ETError ? ETError[errorCode] : ETError[ETError.UndefinedError];
|
|
50
|
+
return ETErrorMessage + message;
|
|
37
51
|
};
|
|
38
52
|
//# sourceMappingURL=Error.js.map
|
package/lib/module/Error.js.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"names":["ETError","getError","e","UndefinedError","error","errorCode","parseInt","message"],"sourceRoot":"../../src","sources":["Error.ts"],"mappings":";;AAAA,WAAYA,OAAO,0BAAPA,OAAO;
|
|
1
|
+
{"version":3,"names":["ETError","getError","e","UndefinedError","error","errorCode","parseInt","message","Number","isNaN","slice","length","trimStart","ETErrorMessage"],"sourceRoot":"../../src","sources":["Error.ts"],"mappings":";;AAAA,WAAYA,OAAO,0BAAPA,OAAO;EACjB;EADUA,OAAO,CAAPA,OAAO;EAAPA,OAAO,CAAPA,OAAO;EAAPA,OAAO,CAAPA,OAAO;EAAPA,OAAO,CAAPA,OAAO;EAAPA,OAAO,CAAPA,OAAO;EAAPA,OAAO,CAAPA,OAAO;EASjB;EATUA,OAAO,CAAPA,OAAO;EAAPA,OAAO,CAAPA,OAAO;EAAPA,OAAO,CAAPA,OAAO;EAcjB;EACA;EACA;EAhBUA,OAAO,CAAPA,OAAO;EAAPA,OAAO,CAAPA,OAAO;EAAPA,OAAO,CAAPA,OAAO;EAAPA,OAAO,CAAPA,OAAO;EAsBjB;EAtBUA,OAAO,CAAPA,OAAO;EAAPA,OAAO,CAAPA,OAAO;EAAPA,OAAO,CAAPA,OAAO;EAAPA,OAAO,CAAPA,OAAO;EAAPA,OAAO,CAAPA,OAAO;EA6BjB;EA7BUA,OAAO,CAAPA,OAAO;EAAPA,OAAO,CAAPA,OAAO;EAAPA,OAAO,CAAPA,OAAO;EAAPA,OAAO,CAAPA,OAAO;EAmCjB;EAnCUA,OAAO,CAAPA,OAAO;EAAPA,OAAO,CAAPA,OAAO;EAAPA,OAAO,CAAPA,OAAO;EAAA,OAAPA,OAAO;AAAA;AAyCnB,OAAO,MAAMC,QAAQ,GAAIC,CAA4B,IAAa;EAChE,IAAI,OAAOA,CAAC,KAAK,QAAQ,EAAE;IACzB,IAAIA,CAAC,IAAIF,OAAO,EAAE,OAAOA,OAAO,CAACE,CAAC,CAAC;IACnC,OAAOF,OAAO,CAACA,OAAO,CAACG,cAAc,CAAC;EACxC;;EAEA;EACA,MAAMC,KAAK,GAAGF,CAAU;EACxB,MAAMG,SAAS,GAAGC,QAAQ,CAACF,KAAK,CAACG,OAAO,EAAE,EAAE,CAAC;EAC7C,MAAMA,OAAO,GAAGC,MAAM,CAACC,KAAK,CAACJ,SAAS,CAAC,GACnCD,KAAK,CAACG,OAAO,GACb,GAAG,GAAGH,KAAK,CAACG,OAAO,CAACG,KAAK,CAAC,GAAGL,SAAS,EAAE,CAACM,MAAM,CAAC,CAACC,SAAS,CAAC,CAAC;EAEhE,MAAMC,cAAc,GAClBR,SAAS,IAAIL,OAAO,GAAGA,OAAO,CAACK,SAAS,CAAC,GAAGL,OAAO,CAACA,OAAO,CAACG,cAAc,CACjE;EAEX,OAAOU,cAAc,GAAGN,OAAO;AACjC,CAAC","ignoreList":[]}
|