react-native-executorch 0.7.0 → 0.7.1

package/third-party/include/executorch/extension/llm/tokenizers/include/pytorch/tokenizers/tokenizer.h

@@ -13,8 +13,8 @@
 
 #pragma once
 
-#include "error.h"
-#include "result.h"
+#include <pytorch/tokenizers/error.h>
+#include <pytorch/tokenizers/result.h>
 #include <string>
 #include <vector>
 

package/third-party/include/executorch/extension/llm/tokenizers/include/pytorch/tokenizers/truncation.h

@@ -0,0 +1,92 @@
+/*
+ * Copyright (c) Meta Platforms, Inc. and affiliates.
+ * All rights reserved.
+ *
+ * This source code is licensed under the BSD-style license found in the
+ * LICENSE file in the root directory of this source tree.
+ */
+// @lint-ignore-every LICENSELINT
+
+#pragma once
+
+// Standard
+#include <memory>
+#include <optional>
+#include <string>
+#include <vector>
+
+// Third Party
+#include <nlohmann/json.hpp>
+
+namespace tokenizers {
+
+// -- Truncation ---------------------------------------------------------------
+
+enum class TruncationStrategy {
+  LongestFirst,
+  OnlyFirst,
+  OnlySecond,
+};
+
+enum class TruncationDirection {
+  Left,
+  Right,
+};
+
+struct TruncationParams {
+  TruncationDirection direction = TruncationDirection::Right;
+  size_t max_length = 512;
+  TruncationStrategy strategy = TruncationStrategy::LongestFirst;
+  size_t stride = 0;
+};
+
+class Truncation {
+public:
+  /** Shared pointer type */
+  typedef std::shared_ptr<Truncation> Ptr;
+
+  /**
+   * @param params: The truncation parameters
+   */
+  explicit Truncation(const TruncationParams &params);
+
+  /**
+   * Truncate the tokens according to the configuration.
+   *
+   * @param tokens The tokens to truncate.
+   * @param num_tokens_to_add The number of special tokens that will be added
+   * later. These are subtracted from max_length during truncation calculation.
+   */
+  std::vector<uint64_t> truncate(std::vector<uint64_t> tokens,
+                                 size_t num_tokens_to_add = 0) const;
+
+  /**
+   * Truncate a pair of sequences according to the configuration.
+   */
+  std::pair<std::vector<uint64_t>, std::vector<uint64_t>>
+  truncate_pair(std::vector<uint64_t> a, std::vector<uint64_t> b,
+                size_t num_tokens_to_add = 0) const;
+
+private:
+  TruncationParams params_;
+};
+
+// -- Factory ------------------------------------------------------------------
+
+class TruncationConfig {
+public:
+  /**
+   * Construct the truncation instance from the member data
+   */
+  Truncation::Ptr create() const;
+
+  /**
+   * Populate from a json config file
+   */
+  TruncationConfig &parse_json(const nlohmann::json &json_config);
+
+  // Configuration members
+  TruncationParams params;
+};
+
+} // namespace tokenizers

package/third-party/include/executorch/extension/llm/tokenizers/include/pytorch/tokenizers/wordpiece_model.h

@@ -0,0 +1,74 @@
+/*
+ * Copyright (c) Meta Platforms, Inc. and affiliates.
+ * All rights reserved.
+ *
+ * This source code is licensed under the BSD-style license found in the
+ * LICENSE file in the root directory of this source tree.
+ */
+// @lint-ignore-every LICENSELINT
+
+#pragma once
+
+#include <memory>
+#include <optional>
+#include <string>
+#include <vector>
+
+#include <pytorch/tokenizers/model.h>
+#include <pytorch/tokenizers/regex.h>
+#include <pytorch/tokenizers/result.h>
+#include <pytorch/tokenizers/string_integer_map.h>
+
+namespace tokenizers {
+
+class WordPieceModel : public Model {
+public:
+  explicit WordPieceModel(detail::TokenMap token_map,
+                          detail::TokenMap special_token_map,
+                          std::string unk_token,
+                          std::string continuing_subword_prefix,
+                          size_t max_input_chars_per_word,
+                          std::optional<uint64_t> unk_token_id,
+                          std::optional<uint64_t> bos_token_id,
+                          std::optional<uint64_t> eos_token_id);
+
+  ~WordPieceModel() override = default;
+
+  Result<std::vector<uint64_t>>
+  tokenize(const std::string &piece) const override;
+
+  Result<std::string> id_to_piece(uint64_t token) const override;
+  Result<uint64_t> piece_to_id(const std::string &token) const override;
+
+  int32_t vocab_size() const override { return vocab_size_; }
+
+  bool is_special_token(uint64_t token) const override;
+
+  bool is_loaded() const override { return initialized_; }
+
+  std::pair<std::optional<std::string>, std::string>
+  split_with_allowed_special_token(const std::string &input,
+                                   size_t offset) const override;
+
+  uint64_t bos_token_id() const override { return bos_token_id_.value_or(0); }
+
+  uint64_t eos_token_id() const override { return eos_token_id_.value_or(0); }
+
+private:
+  detail::TokenMap token_map_;
+  detail::TokenMap special_token_map_;
+  std::unique_ptr<IRegex> special_token_regex_;
+
+  std::string unk_token_;
+  std::string continuing_subword_prefix_;
+  size_t max_input_chars_per_word_;
+
+  std::optional<uint64_t> unk_token_id_;
+  std::optional<uint64_t> bos_token_id_;
+  std::optional<uint64_t> eos_token_id_;
+
+  bool initialized_ = false;
+  int32_t vocab_size_ = 0;
+};
+
+} // namespace tokenizers

package/common/rnexecutorch/tests/CMakeLists.txt

@@ -1,253 +0,0 @@
-if(NOT ANDROID_ABI)
-    message(FATAL_ERROR "Tests can be only built for Android simulator")
-endif()
-
-cmake_minimum_required(VERSION 3.13)
-project(RNExecutorchTests)
-
-set(CMAKE_CXX_STANDARD 20)
-set(CMAKE_CXX_STANDARD_REQUIRED TRUE)
-
-# tests/                         <- CMAKE_SOURCE_DIR (this file's location)
-# rnexecutorch/                  <- RNEXECUTORCH_DIR (parent of tests)
-# common/                        <- COMMON_DIR
-# react-native-executorch/       <- PACKAGE_ROOT
-# <monorepo-root>/               <- MONOREPO_ROOT
-# <monorepo-root>/third-party/   <- THIRD_PARTY_DIR
-set(RNEXECUTORCH_DIR "${CMAKE_SOURCE_DIR}/..")
-set(COMMON_DIR "${RNEXECUTORCH_DIR}/..")
-set(PACKAGE_ROOT "${COMMON_DIR}/..")
-set(MONOREPO_ROOT "${PACKAGE_ROOT}/../..")
-set(THIRD_PARTY_DIR "${MONOREPO_ROOT}/third-party")
-set(REACT_NATIVE_DIR "${MONOREPO_ROOT}/node_modules/react-native")
-set(ANDROID_THIRD_PARTY "${PACKAGE_ROOT}/third-party/android/libs/")
-set(TOKENIZERS_DIR "${PACKAGE_ROOT}/third-party/include/executorch/extension/llm/tokenizers/include")
-
-# Add Gtest as a subdirectory
-add_subdirectory(${THIRD_PARTY_DIR}/googletest ${PROJECT_BINARY_DIR}/googletest)
-
-# ExecuTorch Prebuilt binaries
-add_library(executorch_prebuilt SHARED IMPORTED)
-set_target_properties(executorch_prebuilt PROPERTIES
-    IMPORTED_LOCATION "${ANDROID_THIRD_PARTY}/executorch/${ANDROID_ABI}/libexecutorch.so"
-)
-
-# pthreadpool and cpuinfo (needed for OpenMP/OpenCV)
-if(ANDROID_ABI STREQUAL "arm64-v8a")
-  add_library(pthreadpool SHARED IMPORTED)
-  set_target_properties(pthreadpool PROPERTIES
-      IMPORTED_LOCATION "${ANDROID_THIRD_PARTY}/pthreadpool/${ANDROID_ABI}/libpthreadpool.so"
-  )
-
-  add_library(cpuinfo SHARED IMPORTED)
-  set_target_properties(cpuinfo PROPERTIES
-      IMPORTED_LOCATION "${ANDROID_THIRD_PARTY}/cpuinfo/${ANDROID_ABI}/libcpuinfo.so"
-  )
-
-  set(EXECUTORCH_LIBS pthreadpool cpuinfo)
-else()
-  set(EXECUTORCH_LIBS "")
-endif()
-
-# OpenCV (Interface Library)
-set(OPENCV_LIBS_DIR "${ANDROID_THIRD_PARTY}/opencv/${ANDROID_ABI}")
-set(OPENCV_THIRD_PARTY_DIR "${ANDROID_THIRD_PARTY}/opencv-third-party/${ANDROID_ABI}")
-
-if(ANDROID_ABI STREQUAL "arm64-v8a")
-  set(OPENCV_THIRD_PARTY_LIBS
-    "${OPENCV_THIRD_PARTY_DIR}/libkleidicv_hal.a"
-    "${OPENCV_THIRD_PARTY_DIR}/libkleidicv_thread.a"
-    "${OPENCV_THIRD_PARTY_DIR}/libkleidicv.a"
-  )
-elseif(ANDROID_ABI STREQUAL "x86_64")
-  set(OPENCV_THIRD_PARTY_LIBS "")
-endif()
-
-
-add_library(opencv_deps INTERFACE)
-target_link_libraries(opencv_deps INTERFACE
-    ${OPENCV_LIBS_DIR}/libopencv_core.a
-    ${OPENCV_LIBS_DIR}/libopencv_features2d.a
-    ${OPENCV_LIBS_DIR}/libopencv_highgui.a
-    ${OPENCV_LIBS_DIR}/libopencv_imgproc.a
-    ${OPENCV_LIBS_DIR}/libopencv_photo.a
-    ${OPENCV_LIBS_DIR}/libopencv_video.a
-    ${OPENCV_THIRD_PARTY_LIBS}
-    ${EXECUTORCH_LIBS}
-    z
-    dl
-    m
-    log
-)
-target_link_options(opencv_deps INTERFACE -fopenmp -static-openmp)
-
-add_library(tokenizers_deps INTERFACE)
-target_include_directories(tokenizers_deps INTERFACE "${TOKENIZERS_DIR}")
-
-# Source Definitions
-set(CORE_SOURCES
-    ${RNEXECUTORCH_DIR}/models/BaseModel.cpp
-    ${RNEXECUTORCH_DIR}/data_processing/Numerical.cpp
-    ${CMAKE_SOURCE_DIR}/integration/stubs/jsi_stubs.cpp
-)
-
-set(IMAGE_UTILS_SOURCES
-    ${RNEXECUTORCH_DIR}/data_processing/ImageProcessing.cpp
-    ${RNEXECUTORCH_DIR}/data_processing/base64.cpp
-    ${COMMON_DIR}/ada/ada.cpp
-)
-
-set(TOKENIZER_SOURCES ${RNEXECUTORCH_DIR}/TokenizerModule.cpp)
-set(DSP_SOURCES ${RNEXECUTORCH_DIR}/data_processing/dsp.cpp)
-
-# Core Library
-add_library(rntests_core STATIC ${CORE_SOURCES})
-
-target_include_directories(rntests_core PUBLIC
-    ${RNEXECUTORCH_DIR}/data_processing
-    ${TOKENIZERS_DIR}
-    ${RNEXECUTORCH_DIR}
-    ${COMMON_DIR}
-    ${PACKAGE_ROOT}/third-party/include
-    ${REACT_NATIVE_DIR}/ReactCommon
-    ${REACT_NATIVE_DIR}/ReactCommon/jsi
-    ${REACT_NATIVE_DIR}/ReactCommon/callinvoker
-    ${COMMON_DIR}/ada
-)
-
-target_link_libraries(rntests_core PUBLIC
-    executorch_prebuilt
-    gtest
-    log
-)
-
-enable_testing()
-function(add_rn_test TEST_TARGET TEST_FILENAME)
-    cmake_parse_arguments(ARG "" "" "SOURCES;LIBS" ${ARGN})
-    # Create executable using the explicit filename provided
-    add_executable(${TEST_TARGET} ${TEST_FILENAME} ${ARG_SOURCES})
-
-    target_compile_definitions(${TEST_TARGET} PRIVATE TEST_BUILD)
-    target_link_libraries(${TEST_TARGET} PRIVATE rntests_core gtest_main ${ARG_LIBS})
-    target_link_options(${TEST_TARGET} PRIVATE "LINKER:-z,max-page-size=16384")
-
-    add_test(NAME ${TEST_TARGET} COMMAND ${TEST_TARGET})
-endfunction()
-
-add_rn_test(NumericalTests unit/NumericalTest.cpp)
-add_rn_test(LogTests unit/LogTest.cpp)
-add_rn_test(BaseModelTests integration/BaseModelTest.cpp)
-
-add_rn_test(ClassificationTests integration/ClassificationTest.cpp
-    SOURCES
-        ${RNEXECUTORCH_DIR}/models/classification/Classification.cpp
-        ${IMAGE_UTILS_SOURCES}
-    LIBS opencv_deps
-)
-
-add_rn_test(ObjectDetectionTests integration/ObjectDetectionTest.cpp
-    SOURCES
-        ${RNEXECUTORCH_DIR}/models/object_detection/ObjectDetection.cpp
-        ${RNEXECUTORCH_DIR}/models/object_detection/Utils.cpp
-        ${IMAGE_UTILS_SOURCES}
-    LIBS opencv_deps
-)
-
-add_rn_test(ImageEmbeddingsTests integration/ImageEmbeddingsTest.cpp
-    SOURCES
-        ${RNEXECUTORCH_DIR}/models/embeddings/image/ImageEmbeddings.cpp
-        ${RNEXECUTORCH_DIR}/models/embeddings/BaseEmbeddings.cpp
-        ${IMAGE_UTILS_SOURCES}
-    LIBS opencv_deps
-)
-
-add_rn_test(TextEmbeddingsTests integration/TextEmbeddingsTest.cpp
-    SOURCES
-        ${RNEXECUTORCH_DIR}/models/embeddings/text/TextEmbeddings.cpp
-        ${RNEXECUTORCH_DIR}/models/embeddings/BaseEmbeddings.cpp
-        ${TOKENIZER_SOURCES}
-    LIBS tokenizers_deps
-)
-
-add_rn_test(StyleTransferTests integration/StyleTransferTest.cpp
-    SOURCES
-        ${RNEXECUTORCH_DIR}/models/style_transfer/StyleTransfer.cpp
-        ${IMAGE_UTILS_SOURCES}
-    LIBS opencv_deps
-)
-
-add_rn_test(VADTests integration/VoiceActivityDetectionTest.cpp
-    SOURCES
-        ${RNEXECUTORCH_DIR}/models/voice_activity_detection/VoiceActivityDetection.cpp
-        ${RNEXECUTORCH_DIR}/models/voice_activity_detection/Utils.cpp
-        ${DSP_SOURCES}
-)
-
-add_rn_test(TokenizerModuleTests integration/TokenizerModuleTest.cpp
-    SOURCES ${TOKENIZER_SOURCES}
-    LIBS tokenizers_deps
-)
-
-add_rn_test(SpeechToTextTests integration/SpeechToTextTest.cpp
-    SOURCES
-        ${RNEXECUTORCH_DIR}/models/speech_to_text/SpeechToText.cpp
-        ${RNEXECUTORCH_DIR}/models/speech_to_text/asr/ASR.cpp
-        ${RNEXECUTORCH_DIR}/models/speech_to_text/stream/HypothesisBuffer.cpp
-        ${RNEXECUTORCH_DIR}/models/speech_to_text/stream/OnlineASRProcessor.cpp
-        ${RNEXECUTORCH_DIR}/data_processing/gzip.cpp
-        ${TOKENIZER_SOURCES}
-        ${DSP_SOURCES}
-    LIBS tokenizers_deps z
-)
-
-add_rn_test(LLMTests integration/LLMTest.cpp
-    SOURCES
-        ${RNEXECUTORCH_DIR}/models/llm/LLM.cpp
-        ${COMMON_DIR}/runner/runner.cpp
-        ${COMMON_DIR}/runner/text_prefiller.cpp
-        ${COMMON_DIR}/runner/text_decoder_runner.cpp
-        ${COMMON_DIR}/runner/sampler.cpp
-        ${COMMON_DIR}/runner/arange_util.cpp
-    LIBS tokenizers_deps
-)
-
-add_rn_test(TextToImageTests integration/TextToImageTest.cpp
-    SOURCES
-        ${RNEXECUTORCH_DIR}/models/text_to_image/TextToImage.cpp
-        ${RNEXECUTORCH_DIR}/models/text_to_image/Encoder.cpp
-        ${RNEXECUTORCH_DIR}/models/text_to_image/UNet.cpp
-        ${RNEXECUTORCH_DIR}/models/text_to_image/Decoder.cpp
-        ${RNEXECUTORCH_DIR}/models/text_to_image/Scheduler.cpp
-        ${RNEXECUTORCH_DIR}/models/embeddings/text/TextEmbeddings.cpp
-        ${RNEXECUTORCH_DIR}/models/embeddings/BaseEmbeddings.cpp
-        ${TOKENIZER_SOURCES}
-    LIBS tokenizers_deps
-)
-
-add_rn_test(OCRTests integration/OCRTest.cpp
-    SOURCES
-        ${RNEXECUTORCH_DIR}/models/ocr/OCR.cpp
-        ${RNEXECUTORCH_DIR}/models/ocr/CTCLabelConverter.cpp
-        ${RNEXECUTORCH_DIR}/models/ocr/Detector.cpp
-        ${RNEXECUTORCH_DIR}/models/ocr/RecognitionHandler.cpp
-        ${RNEXECUTORCH_DIR}/models/ocr/Recognizer.cpp
-        ${RNEXECUTORCH_DIR}/models/ocr/utils/DetectorUtils.cpp
-        ${RNEXECUTORCH_DIR}/models/ocr/utils/RecognitionHandlerUtils.cpp
-        ${RNEXECUTORCH_DIR}/models/ocr/utils/RecognizerUtils.cpp
-        ${IMAGE_UTILS_SOURCES}
-    LIBS opencv_deps
-)
-
-add_rn_test(VerticalOCRTests integration/VerticalOCRTest.cpp
-    SOURCES
-        ${RNEXECUTORCH_DIR}/models/vertical_ocr/VerticalOCR.cpp
-        ${RNEXECUTORCH_DIR}/models/vertical_ocr/VerticalDetector.cpp
-        ${RNEXECUTORCH_DIR}/models/ocr/Detector.cpp
-        ${RNEXECUTORCH_DIR}/models/ocr/CTCLabelConverter.cpp
-        ${RNEXECUTORCH_DIR}/models/ocr/Recognizer.cpp
-        ${RNEXECUTORCH_DIR}/models/ocr/utils/DetectorUtils.cpp
-        ${RNEXECUTORCH_DIR}/models/ocr/utils/RecognitionHandlerUtils.cpp
-        ${RNEXECUTORCH_DIR}/models/ocr/utils/RecognizerUtils.cpp
-        ${IMAGE_UTILS_SOURCES}
-    LIBS opencv_deps
-)

package/common/rnexecutorch/tests/README.md

@@ -1,73 +0,0 @@
-## Native Test
-This guide provide information on how functions are tested, how to install all needed dependencies and how to run tests.
-
-### Used Tools
-To test the native code we use [`googletest`](https://github.com/google/googletest). It's a flexible tool for creating unit tests.
-
-### Installation
-The googletest is already in repo in `react-native-executorch/third-party/googletest`. Firstly, you need to fetch googletest locally, run from root directory of project:
-* `git submodule update --init --recursive third-party/googletest`
-
-### Running tests
-
-#### Prerequisites
-
-- **Android NDK**: The `ANDROID_NDK` environment variable must be set
-- **wget**: Must be in your PATH
-- **Android emulator**: Must be running before executing tests
-- **Device requirements**:
-  - 16GB disk storage (minimum)
-  - 8GB RAM (minimum)
-
-#### First-time setup
-
-Before running tests, you need to build an app to generate required native libraries (`libfbjni.so` and `libc++_shared.so`). The test script automatically searches for these in the monorepo.
-
-If the script reports missing libraries, build any example app:
-```bash
-cd apps/computer-vision/android
-./gradlew assembleDebug
-# or
-./gradlew assembleRelease
-```
-
-#### Running the tests
-
-Navigate to the tests directory:
-```bash
-cd packages/react-native-executorch/common/rnexecutorch/tests
-```
-
-Run the test script:
-```bash
-bash ./run_tests.sh
-```
-
-This script:
-- Downloads all needed models
-- Pushes executables, models, assets, and shared libraries via ADB to the running emulator
-- Runs the pre-compiled test executables
-
-#### Available flags
-
-* `--refresh-models` - Forcefully downloads all the models. By default, models are not downloaded unless they are missing from the specified directory.
-* `--skip-build` - Skips the cmake build step.
-
-### How to add a new test
-To add new test you need to:
-* Add a new .cpp file to either integration/ or unit/, depending on the type of the test.
-* In `CMakeLists.txt`, add all executables and link all the needed libraries against the executable, for example you can use the `add_rn_test`, which is a helper function that links core libs. Example:
-    ```cmake
-    # unit
-    add_rn_test(BaseModelTests integration/BaseModelTest.cpp)
-
-    # integration
-    add_rn_test(ClassificationTests integration/ClassificationTest.cpp
-        SOURCES
-            ${RNEXECUTORCH_DIR}/models/classification/Classification.cpp
-            ${IMAGE_UTILS_SOURCES}
-        LIBS opencv_deps
-    )
-    ```
-* Lastly, add the test executable name to the run_tests script along with all the needed URL and assets.
-