react-native-executorch 0.10.5 → 0.10.7

package/src/fetcher/fetcher.ts

@@ -14,9 +14,11 @@ const IS_ANDROID = Platform.OS === 'android';
 // Persistent, per-app directory where downloaded model assets are cached.
 //   iOS: internal DocumentDir (not CacheDir) so the OS won't evict large models
 //        between runs and force a costly re-download.
-//   Android: the app-private EXTERNAL files dir (getExternalFilesDir), so the
-//        system DownloadManager can write there and same-volume moves stay cheap
-//        even for multi-GB files. Falls back to DocumentDir if unmounted.
+//   Android: the app-private EXTERNAL files dir (getExternalFilesDir). It was
+//        picked so the system DownloadManager could stage into it; that backend
+//        is gone, but every model already on disk lives here, so moving the
+//        cache now would silently orphan all of them and re-download multi-GB
+//        files. Falls back to DocumentDir if unmounted.
 const ANDROID_DIRECTORY = RNBlobUtil.fs.dirs.SDCardDir || RNBlobUtil.fs.dirs.DocumentDir;
 const RNE_DIRECTORY = IS_ANDROID
   ? `${ANDROID_DIRECTORY}/react-native-executorch`
@@ -31,9 +33,7 @@ export interface DownloadOptions {
   onProgress?: (progress: number) => void;
   /**
    * Aborts the download. The bytes fetched so far are kept so a later
-   * {@link download} of the same source resumes instead of restarting, except on
-   * Android without the optional background downloader, where the system
-   * DownloadManager discards a cancelled transfer.
+   * {@link download} of the same source resumes instead of restarting.
    */
   signal?: AbortSignal;
   /**
@@ -176,24 +176,6 @@ async function foldResumedChunkIntoPartial(
   await RNBlobUtil.fs.unlink(chunkPath).catch(() => {});
 }
 
-// DownloadManager's byte counter is 64-bit, but blob-util reads it out of the
-// cursor with `getInt`, so what reaches JS is the low 32 bits reinterpreted as
-// a signed int: past 2 GB it arrives NEGATIVE and wraps every 4 GB after that.
-// Multi-GB LLM models spend most of their download inside that range, which is
-// what collapsed their progress bar. The counter only ever grows, so the
-// discarded high bits can be rebuilt by counting how often the low ones wrap.
-const UINT32 = 0x100000000;
-function reassemble32BitCounter(): (raw: number) => number {
-  let wraps = 0;
-  let previous = 0;
-  return (raw) => {
-    const low = raw < 0 ? raw + UINT32 : raw;
-    if (low < previous) wraps += 1;
-    previous = low;
-    return low + wraps * UINT32;
-  };
-}
-
 // Reports absolute bytes for one file. `total` is 0 when the transfer does not
 // know the length yet — the receiver keeps using whatever length it already
 // had rather than treating the file as complete.
@@ -249,8 +231,7 @@ async function downloadUrl(url: string, cb: DownloadUrlCallbacks): Promise<strin
     // something a later download would CONTINUE from: the staged `.partial` and,
     // with the background downloader in play, a paused task holding resume data.
     // Clear it, or "download it again" quietly resumes the very attempt the
-    // caller is trying to replace. The DownloadManager backend needs nothing
-    // here — it unlinks its own staging file before every transfer.
+    // caller is trying to replace.
     await discardPartialDownload(dest);
   } else if (await RNBlobUtil.fs.exists(dest)) {
     // Cache hit — nothing to download.
@@ -296,30 +277,19 @@ async function startDownload(url: string, dest: string, entry: InFlightDownload)
   };
 
   // The optional background downloader wins on BOTH platforms when the app has
-  // it, so a caller that installs it gets one transfer mechanism that behaves
-  // the same everywhere instead of a per-platform one it cannot see.
-  //
-  // Without it each platform falls back to the best it can do on its own, and
-  // only iOS loses background transfers by doing so — blob-util's in-process
-  // reader is broken on Android (RonRadtke/react-native-blob-util#475: it stops
-  // after 8 KB), so the system DownloadManager is not a preference there but the
-  // only backend that works at all.
+  // it; without it both fall back to the same in-process stream. One mechanism
+  // plus one shared fallback, so a caller sees the same behavior everywhere.
   const backgroundDownloader = loadBackgroundDownloader();
 
-  let path: string;
-  if (backgroundDownloader) {
-    path = await downloadUrlViaBackgroundSession(backgroundDownloader, url, dest, cb);
-  } else if (IS_ANDROID) {
-    path = await downloadUrlViaAndroidDownloadManager(url, dest, cb);
-  } else {
-    path = await downloadUrlViaIosStream(url, dest, cb);
-  }
+  const path = backgroundDownloader
+    ? await downloadUrlViaBackgroundSession(backgroundDownloader, url, dest, cb)
+    : await downloadUrlViaStream(url, dest, cb);
 
-  // Neither backend is guaranteed to emit a last sample at 100%: blob-util
-  // throttles progress events, and DownloadManager is polled, so the final
-  // chunk usually lands between two polls. Without this a finished file stays
-  // stuck a little short of its own size, and in a multi-file config that
-  // shortfall is what the user sees while the remaining files download.
+  // Neither backend is guaranteed to emit a last sample at 100%: blob-util only
+  // started reporting the final event reliably in 0.25.0, and the background
+  // downloader throttles its own. Without this a finished file stays stuck a
+  // little short of its own size, and in a multi-file config that shortfall is
+  // what the user sees while the remaining files download.
   const finalSize = await fileSize(path);
   if (finalSize > 0) cb.onBytes?.(finalSize, finalSize);
 
@@ -347,72 +317,6 @@ function joinDownload(entry: InFlightDownload, cb: DownloadUrlCallbacks): Promis
   });
 }
 
-// Android fallback used when that optional dependency is absent: the system
-// DownloadManager streams to app-private external storage. blob-util's
-// in-process reader cannot stand in for it — upstream #475 makes that path stop
-// after 8 KB — and DownloadManager also handles files larger than 2 GB, keeps
-// downloading while the app is in the background or killed, and resumes across
-// transient network drops on its own, so no manual Range logic.
-async function downloadUrlViaAndroidDownloadManager(
-  url: string,
-  dest: string,
-  cb: DownloadUrlCallbacks
-): Promise<string> {
-  const tmp = `${dest}.downloading`;
-  await RNBlobUtil.fs.unlink(tmp).catch(() => {});
-
-  if (cb.signal?.aborted) throw abortError();
-
-  const task = RNBlobUtil.config({
-    addAndroidDownloads: {
-      useDownloadManager: true,
-      path: tmp,
-      notification: false,
-      mediaScannable: false,
-      mime: 'application/octet-stream',
-    },
-  }).fetch('GET', url);
-
-  const onAbort = () => task.cancel();
-  cb.signal?.addEventListener('abort', onAbort);
-
-  // DownloadManager reports total as -1 until the size is known; pass that on
-  // as 0 rather than echoing the received count, which would otherwise look
-  // like a file that is complete at every sample.
-  const absoluteBytes = reassemble32BitCounter();
-  task.progress({ count: 100 }, (received, total) => {
-    const tot = Number(total);
-    cb.onBytes?.(absoluteBytes(Number(received)), tot > 0 ? tot : 0);
-  });
-
-  try {
-    await task;
-  } catch (e) {
-    await RNBlobUtil.fs.unlink(tmp).catch(() => {});
-    throw cb.signal?.aborted ? abortError() : e;
-  } finally {
-    cb.signal?.removeEventListener('abort', onAbort);
-  }
-
-  // DownloadManager doesn't surface an HTTP status; an empty file means failure.
-  const size = await fileSize(tmp);
-  if (size <= 0) {
-    await RNBlobUtil.fs.unlink(tmp).catch(() => {});
-    throw RnExecuTorchError('DOWNLOAD_FAILED', `Download of ${url} failed (empty response).`);
-  }
-  // A non-empty file is not necessarily a complete one, and DownloadManager
-  // gives us no status to check. Promoting a short file would cache it under
-  // its final name forever — the existence-only cache check can't tell the
-  // difference, and a truncated .pte only fails much later, at load.
-  const expected = await expectedBytesFor(url, cb.expectedBytes, cb.signal);
-  if (isTruncated(size, expected)) {
-    await RNBlobUtil.fs.unlink(tmp).catch(() => {});
-    throw incompleteError(url, size, expected);
-  }
-  await RNBlobUtil.fs.mv(tmp, dest);
-  return dest;
-}
-
 // One background task per destination file, under an id that stays the same
 // across app launches: that is what lets a later call adopt a transfer this
 // process never started.
@@ -579,14 +483,19 @@ async function downloadUrlViaBackgroundSession(
   return dest;
 }
 
-// iOS fallback used when that optional dependency is absent: blob-util streams
-// via the iOS URL session straight to disk. It does NOT survive the app being
-// suspended — iOS tears the connection down about a second later — so an
-// interrupted transfer is picked up by the next `download` call instead.
-// Interrupted downloads resume from a `.partial` file via an HTTP Range request.
-// `canResume` is set to `false` on an internal retry to avoid recursing forever
-// if partial-file assembly ever fails.
-async function downloadUrlViaIosStream(
+// The fallback used on either platform when that optional dependency is absent:
+// blob-util streams the response straight to disk, in process. It does NOT
+// survive the app being suspended — iOS tears the connection down about a
+// second later, and Android may kill the process outright — so an interrupted
+// transfer is picked up by the next `download` call instead: it resumes from a
+// `.partial` file via an HTTP Range request. `canResume` is set to `false` on an
+// internal retry to avoid recursing forever if partial-file assembly ever fails.
+//
+// This path needs react-native-blob-util >=0.24.11 on Android. 0.24.10's reader
+// stopped after exactly 8 KB with "Download interrupted"
+// (RonRadtke/react-native-blob-util#475), which is why Android used the system
+// DownloadManager until that fix shipped.
+async function downloadUrlViaStream(
   url: string,
   dest: string,
   cb: DownloadUrlCallbacks,
@@ -638,9 +547,9 @@ async function downloadUrlViaIosStream(
     });
   }
 
-  // Same granularity as Android. blob-util still floors the rate at one event
-  // per 250 ms, so this only means a large file advances in ~1% steps instead
-  // of the 5% ones that made a multi-GB download look stalled between jumps.
+  // blob-util floors the rate at one event per 250 ms regardless, so this only
+  // means a large file advances in ~1% steps instead of the 5% ones that made a
+  // multi-GB download look stalled between jumps.
   task.progress({ count: 100 }, (received, total) => {
     const recv = Number(received);
     const tot = Number(total);
@@ -733,14 +642,14 @@ async function downloadUrlViaIosStream(
     // once as a plain full download so correctness never depends on resume.
     await RNBlobUtil.fs.unlink(part).catch(() => {});
     await RNBlobUtil.fs.unlink(target).catch(() => {});
-    if (canResume) return downloadUrlViaIosStream(url, dest, cb, false);
+    if (canResume) return downloadUrlViaStream(url, dest, cb, false);
     throw assemblyErr;
   }
 
   if (restart) {
     await RNBlobUtil.fs.unlink(part).catch(() => {});
     await RNBlobUtil.fs.unlink(target).catch(() => {});
-    return downloadUrlViaIosStream(url, dest, cb, false);
+    return downloadUrlViaStream(url, dest, cb, false);
   }
 
   // The transfer can also report success while the body was cut short — a
@@ -756,7 +665,7 @@ async function downloadUrlViaIosStream(
     // start over rather than fail. A fresh transfer that overshoots is a wrong
     // expectation instead, and falls through to be accepted.
     await RNBlobUtil.fs.unlink(part).catch(() => {});
-    return downloadUrlViaIosStream(url, dest, cb, false);
+    return downloadUrlViaStream(url, dest, cb, false);
   }
   if (isTruncated(assembled, expected)) {
     // Short: keep the partial so the next call resumes and finishes it.
@@ -859,10 +768,8 @@ function substituteRemoteSources<T>(node: T, resolved: ReadonlyMap<string, strin
  * and survive it being killed. The fetcher uses it automatically on both platforms
  * when it is present, so the behavior is the same on each; nothing else changes.
  *
- * Without it the fetcher falls back to what each platform can do on its own: the
- * system DownloadManager on Android, which still continues in the background,
- * and on iOS a streaming request that stops when the app is suspended and is
- * resumed by the next `download` call.
+ * Without it both platforms fall back to the same streaming request, which stops
+ * when the app is suspended and is resumed by the next `download` call.
  * @category Utils / Functions
  * @typeParam T The shape of the value being resolved.
  * @param source A URL, a local path, or any nested object/array holding them.

package/src/fetcher/telemetry.ts

@@ -10,7 +10,7 @@ const DOWNLOAD_EVENT_ENDPOINT = 'https://ai.swmansion.com/telemetry/downloads/ap
 // self-referencing import would need package `exports` support in the consuming
 // bundler to load at all. A wrong value here is analytics noise; a failed
 // import would break the bundle.
-const LIB_VERSION = '0.10.5';
+const LIB_VERSION = '0.10.7';
 
 // Anonymous analytics are on by default; apps opt out via setTelemetryEnabled.
 let telemetryEnabled = true;

package/src/models.ts

@@ -922,10 +922,15 @@ const LFM2_5_EMBEDDING_350M_MLX_INT4: TextEmbedderModel = {
 // =============================================================================
 // Image Embeddings
 // =============================================================================
+// The exported image towers expect CLIP-normalized pixels (mean
+// [0.48145466, 0.4578275, 0.40821073], std [0.26862954, 0.26130258, 0.27577711]).
 const CLIP_IMAGE_EMBEDDINGS_OPTS = {
   resizeMode: 'stretch' as const,
   interpolation: 'linear' as const,
-  normalizeOpts: { alpha: 1 / 255.0, beta: 0.0 },
+  normalizeOpts: {
+    alpha: [1 / (255.0 * 0.26862954), 1 / (255.0 * 0.26130258), 1 / (255.0 * 0.27577711)],
+    beta: [-0.48145466 / 0.26862954, -0.4578275 / 0.26130258, -0.40821073 / 0.27577711],
+  },
 };
 const CLIP_VIT_BASE_PATCH32_IMAGE_XNNPACK_FP32: ImageEmbedderModel = {
   modelPath: `${BASE_URL}-clip-vit-base-patch32/${NEXT_VERSION_TAG}/xnnpack/clip_vit_base_patch32_image_xnnpack_fp32.pte`,

package/third-party/common/phonemis/CMakeLists.txt

@@ -1,45 +1,101 @@
+# ------------------------
+# --- Project metadata ---
+# ------------------------
 cmake_minimum_required(VERSION 3.10)
 project(phonemis VERSION 1.0 LANGUAGES CXX)
 
+# ---------------------
 # --- Configuration ---
+# ---------------------
 set(CMAKE_CXX_STANDARD 20)
 set(CMAKE_CXX_STANDARD_REQUIRED ON)
 
+# By default, we only build a static library.
+# Building the tests (-DBUILD_TESTS=ON) and building the sample runner (-DBUILD_RUNNER=ON) must
+# be explciitely specified as CMAKE options.
 option(BUILD_RUNNER "Build phonemis runner" OFF)
 option(BUILD_TESTS "Build tests" OFF)
 
+# Additional option to specify the usage of ExecuTorch for neural model inference.
+# To use it, you must explicitely link the package with ExecuTorch libs or
+# build it together.
+option(ET_ON "Use ExecuTorch" OFF)
+
+# ----------------
 # --- Includes ---
+# ----------------
 include_directories(${CMAKE_CURRENT_SOURCE_DIR}/src)
 include_directories(${CMAKE_CURRENT_SOURCE_DIR}/src/third-party)
 
+# --------------------
 # --- Source Files ---
+# --------------------
 file(GLOB_RECURSE LIB_SOURCES CONFIGURE_DEPENDS
-    ${CMAKE_CURRENT_SOURCE_DIR}/src/phonemis/*.cpp
+  ${CMAKE_CURRENT_SOURCE_DIR}/src/phonemis/*.cpp
 )
 # Exclude main.cpp from the static library
 list(FILTER LIB_SOURCES EXCLUDE REGEX "main\\.cpp$")
 
-# --- Targets ---
+# Exclude protophone sources when ExecuTorch is used
+if(ET_ON)
+  list(FILTER LIB_SOURCES EXCLUDE REGEX "${CMAKE_CURRENT_SOURCE_DIR}/src/phonemis/protophone/")
+endif()
 
+# ---------------
+# --- Targets ---
+# ---------------
 # Always build the static library
 add_library(phonemis STATIC ${LIB_SOURCES})
 
+if(ET_ON)
+  target_compile_definitions(phonemis PRIVATE ET_ON)
+endif()
+
+# Threading support (std::thread in the Protophone convolution stage).
+find_package(Threads REQUIRED)
+target_link_libraries(phonemis PUBLIC Threads::Threads)
+
+# SIMD architecture selection (xsimd)
+# xsimd picks its widest batch type at compile time from the enabled ISA. ARM
+# targets (Android arm64-v8a, iOS arm64) already imply NEON, so nothing is
+# needed there. For native x86_64 desktop builds we opt into AVX2 + FMA. The
+# flag is PUBLIC so every translation unit (and the tests) share the same
+# default arch, keeping xsimd's alignment/ABI consistent across the boundary.
+if(NOT ET_ON AND NOT ANDROID AND NOT CMAKE_SYSTEM_NAME STREQUAL "iOS"
+   AND CMAKE_SYSTEM_PROCESSOR MATCHES "(x86_64|AMD64|amd64)")
+  include(CheckCXXCompilerFlag)
+  check_cxx_compiler_flag("-mavx2" PHONEMIS_HAS_AVX2)
+  check_cxx_compiler_flag("-mfma" PHONEMIS_HAS_FMA)
+  if(PHONEMIS_HAS_AVX2)
+    target_compile_options(phonemis PUBLIC -mavx2)
+  endif()
+  if(PHONEMIS_HAS_FMA)
+    target_compile_options(phonemis PUBLIC -mfma)
+  endif()
+endif()
+
 # Build runner if requested
 if(BUILD_RUNNER)
-    set(MAIN_SOURCE ${CMAKE_CURRENT_SOURCE_DIR}/src/phonemis/main.cpp)
-    add_executable(phonemis_runner ${MAIN_SOURCE})
-    target_link_libraries(phonemis_runner PRIVATE phonemis)
+  set(MAIN_SOURCE ${CMAKE_CURRENT_SOURCE_DIR}/src/phonemis/main.cpp)
+  add_executable(phonemis_runner ${MAIN_SOURCE})
+  target_link_libraries(phonemis_runner PRIVATE phonemis)
 endif()
 
 # Build tests if requested
 if(BUILD_TESTS)
-    file(GLOB TEST_SOURCES ${CMAKE_CURRENT_SOURCE_DIR}/test/*.cpp)
-    
-    add_executable(phonemis_test ${TEST_SOURCES})
-    target_link_libraries(phonemis_test PRIVATE phonemis)
-    
-    target_include_directories(phonemis_test PRIVATE ${CMAKE_CURRENT_SOURCE_DIR}/test)
-    target_include_directories(phonemis_test PRIVATE ${CMAKE_CURRENT_SOURCE_DIR}/src)
-    target_compile_definitions(phonemis_test PRIVATE PHONEMIS_PROJECT_ROOT="${CMAKE_CURRENT_SOURCE_DIR}")
-    target_compile_options(phonemis_test PRIVATE -Wno-deprecated-declarations)
+  file(GLOB TEST_SOURCES ${CMAKE_CURRENT_SOURCE_DIR}/test/*.cpp)
+  
+  add_executable(phonemis_test ${TEST_SOURCES})
+  target_link_libraries(phonemis_test PRIVATE phonemis)
+  
+  target_include_directories(phonemis_test PRIVATE ${CMAKE_CURRENT_SOURCE_DIR}/test)
+  target_include_directories(phonemis_test PRIVATE ${CMAKE_CURRENT_SOURCE_DIR}/src)
+  target_compile_definitions(phonemis_test PRIVATE PHONEMIS_PROJECT_ROOT="${CMAKE_CURRENT_SOURCE_DIR}")
+  target_compile_options(phonemis_test PRIVATE -Wno-deprecated-declarations)
+
+  # Always add previously excluded Protophone sources.
+  if(ET_ON)
+    file(GLOB_RECURSE PROTOPHONE_SOURCES ${CMAKE_CURRENT_SOURCE_DIR}/src/phonemis/protophone/*.cpp)
+    target_sources(phonemis_test PRIVATE ${PROTOPHONE_SOURCES})
+  endif()
 endif()

package/third-party/common/phonemis/README.md

@@ -1,29 +1,54 @@
 # Phonemis
 
-![Status: In Development](https://img.shields.io/badge/status-in%20development-green)
-
 ## From Text to Sound
-Phonemis is a high-performance C++ library designed for **Grapheme-to-Phoneme (G2P)** conversion. It provides a robust pipeline for transforming raw text into *phonetic transcriptions* using the **International Phonetic Alphabet (IPA)**. The library is optimized for efficiency and portability, being pure C++ with no external dependencies, which makes it easy to implement on a wide range of systems including standard and mobile applications requiring text-to-speech frontend processing.
+
+Phonemis is a high-performance C++ library for **Grapheme-to-Phoneme (G2P)** conversion. It
+delivers universal **IPA phonemization** across many platforms with **zero external
+dependencies**, making it an ideal frontend for text-to-speech systems on desktop, mobile, and
+embedded devices.
 
 Currently supported languages:
-*   🇺🇸 English (US)
-*   🇬🇧 English (British)
 
-## The Mechanics of Pronunciation
-The phonemization pipeline consists of several distinct stages designed to maximize accuracy and context awareness:
+*  🇺🇸 English (US) — `en-us`
+*  🇬🇧 English (British) — `en-gb`
+*  🇩🇪 German — `de`
+*  🇫🇷 French — `fr`
+*  🇪🇸 Spanish — `es`
+*  🇮🇹 Italian — `it`
+*  🇵🇱 Polish — `pl`
+*  🇵🇹 Portuguese — `pt`
+*  🇮🇳 Hindi — `hi`
 
-1.  **Preprocessing**: Raw input text is normalized to handle encoding issues and standard formatting.
-2.  **Rule-based Tokenizer**: The text is segmented into tokens based on linguistic rules, separating words from punctuation and handling special cases.
-3.  **Part-of-Speech Tagging**: A Hidden Markov Model (HMM) bigram tagger is employed to assign grammatical categories to words. This model is trained on the Brown Corpus to resolve homograph ambiguities based on context.
-4.  **Viterbi Decoding**: The optimal sequence of tags is determined using the [Viterbi algorithm](https://en.wikipedia.org/wiki/Viterbi_algorithm), ensuring the most probable grammatical structure is selected.
-5.  **Lexicon-based Phonemization**: Words are converted to phonemes using extensive dictionaries, with fallback mechanisms for unknown tokens.
+## Repository Structure
 
-This library is inspired by the Python package [misaki](https://github.com/hexgrad/misaki).
+```
+phonemis/
+├── data/                          # Bundled resources (one subdirectory per language)
+├── src/
+│   ├── phonemis/
+│   │   ├── base/                  # Core API: Pipeline, IPipeline, Config, phonemizer, tokenizer
+│   │   ├── lang/                  # Language-specific modules (en/, de/, fr/, …)
+│   │   ├── protophone/            # Pure C++ neural model inference engine
+│   │   └── utils/                 # Conversions, string utilities, Unicode support
+│   └── third-party/               # Bundled header-only dependencies (xsimd)
+└── CMakeLists.txt
+```
+
+Every language folder under `data/` contains the resources needed for phonemization — a lexicon
+JSON file (word-to-phoneme dictionary), a Protophone model weights file (`.bin`) for neural
+inference, and optionally a tagger JSON file for part-of-speech disambiguation (currently
+available for English). These are the files you point to in `phonemizer::Config`.
 
 ## Installation
 
-### Building with CMake
-Phonemis uses CMake as its build system. To build the static library:
+### Requirements
+- **C++20** compiler
+- **CMake** ≥ 3.10
+- **xsimd** (bundled as a header-only library under `src/third-party/`)
+
+### Building the Library
+
+The below script builds the package as a static library:
 
 ```bash
 mkdir build
@@ -32,39 +57,164 @@ cmake ..
 make
 ```
 
+### Build Options
+
+| Option | Default | Description |
+|--------|---------|-------------|
+| `-DBUILD_RUNNER=ON` | OFF | Builds the `phonemis_runner` CLI tool |
+| `-DBUILD_TESTS=ON` | OFF | Builds the test suite |
+| `-DET_ON=ON` | OFF | Enables ExecuTorch inference (see below) |
+
+### ExecuTorch Integration
+
+With `ET_ON=ON`, the built-in Protophone sources are excluded and neural phonemization is
+delegated to ExecuTorch. You must link against the ExecuTorch runtime:
+
+```bash
+cmake .. -DET_ON=ON -DCMAKE_PREFIX_PATH=/path/to/executorch
+make
+```
+
+When `ET_ON` is left OFF (the default), all neural inference runs through the built-in
+Protophone model — no external runtime dependencies required.
+
+### SIMD Acceleration
+
+SIMD is configured automatically at build time:
+- **x86_64 desktop** (non-Emscripten, non-Android, non-iOS): AVX2 + FMA are enabled if the
+  compiler supports them (`-mavx2 -mfma`), accelerating Protophone's convolution and mixing
+  stages.
+- **ARM** (Android arm64-v8a, iOS arm64): NEON is implicitly available and used by xsimd with
+  no extra flags.
+
 ### Mobile Builds
-The repository includes dedicated scripts for cross-compiling the library for mobile platforms:
-*   **Android**: Use the provided Android build script to generate `.a` libraries for various ABIs (armeabi-v7a, arm64-v8a, x86, x86_64).
-*   **iOS**: Use the iOS build script to generate a universal static library or framework.
+
+Dedicated scripts are provided for cross-compiling:
+- **Android**: Produces `.a` libraries for armeabi-v7a, arm64-v8a, x86, and x86_64 ABIs.
+- **iOS**: Produces a universal static library or framework.
 
 ## Sample Usage
 
-Below is a minimalistic example demonstrating how to instantiate the pipeline and process text.
+All lexicons and trained neural model weights are bundled in the `./data/` subdirectory, organized
+by language code. The example below uses the English (US) resources:
 
 ```cpp
-#include <phonemis/pipeline.h>
-#include <phonemis/utilities/string_utils.h>
+#include <phonemis/base/pipeline.h>
+#include <phonemis/utils/conversions.h>
 #include <iostream>
 
-using namespace phonemis;
-using namespace phonemis::utilities;
-
 int main() {
-    // Paths to required data files
-    std::string tagger_path = "../data/hmm.json";
-    std::string lexicon_path = "../data/dictionaries/us_merged.json";
+    using namespace phonemis;
+
+    phonemizer::Config phon_config;
+    phon_config.lang = "en-us";
+    phon_config.lexicon_filepath = "data/en-us/lexicon_full.json";
+    phon_config.nn_model_filepath = "data/en-us/phonemizer_en_us.bin";
+
+    Config config;
+    config.lang = "en-us";
+    config.phonemizer = phon_config;
 
-    // Initialize pipeline for US English
-    Pipeline pipeline(Lang::EN_US, tagger_path, lexicon_path);
+    Pipeline pipeline(config);
 
-    // Process text
     std::string text = "I love it! This is the best day of my entire life.";
-    auto phonemes = pipeline.process(text);
+    auto phonemes = pipeline(text);
 
-    // Output result
     std::cout << "Text: " << text << "\n";
-    std::cout << "Phonemes: " << string_utils::u32string_to_utf8(phonemes) << "\n";
+    std::cout << "Phonemes: " << utils::conversions::u32_to_utf8(phonemes) << "\n";
 
     return 0;
 }
 ```
+
+The `Pipeline::operator()` accepts both `std::string_view` (UTF-8) and `std::u32string_view`
+(UTF-32), returning phonemes as `std::u32string`. Individual pipeline stages —
+`preprocess()`, `process()`, `postprocess()` — are also exposed.
+
+### Customizing preprocessing
+
+Before phonemization, the text goes through layers which read it out loud: character and
+whitespace normalization, abbreviations and web addresses, numbers, symbols, and character fixes. Each can be
+turned off, and a language's own abbreviations and symbols can be extended or overridden:
+
+```cpp
+Config config;
+config.lang = "en-us";
+config.phonemizer = phon_config;
+config.preprocessor = {
+    .symbols = false,                                    // Keep "&", "+", ... as written
+    .abbreviations = {{U"approx.", {U"approximately"}}}, // Added to the language's own
+};
+```
+
+| Field | Layer | Example |
+|---|---|---|
+| `normalize` | one form of each character | `’` → `'`, `„a“` → `"a"`, `a – b` → `a - b`, `…` → `...` |
+| `trim` | whitespace normalization | `"  a   b , c"` → `"a b, c"` |
+| `terms` | abbreviations, web addresses | `e.g.` → *for example*, `example.com` → *example dot com* |
+| `numbers` | numbers, dates, money, units, ... | `$5` → *five dollars* |
+| `symbols` | remaining symbols | `&` → *and* |
+| `sanitize` | language's character fixes, if it has any | |
+| `abbreviations`, `symbol_words` | extra table entries | `{U"™", {U"trademark"}}` |
+
+All layers are on by default, and always run in this order: normalize, trim, terms, numbers, symbols, sanitize.
+
+### CLI Runner
+
+```bash
+./build/phonemis_runner --lang en-us \
+    --lexicon data/en-us/lexicon_full.json \
+    --model data/en-us/phonemizer_en_us.bin \
+    "Hello world"
+```
+
+## The Mechanics of Pronunciation
+
+Phonemis combines three complementary phonemization strategies, selected per word to maximize
+accuracy:
+
+### Rule-based
+
+A language-aware tokenizer segments text using configurable rules for words, punctuation, and
+special characters. A number-to-word layer verbalizes digits, dates, currencies, fractions, and
+ordinals into their textual forms according to language-specific conventions (e.g., German
+"einundzwanzig" unit-before-tens order, comma decimal separator).
+
+### Lexicon (lookup-based)
+
+An O(1) dictionary lookup provides the fastest and most reliable phonemization. It is cheap, can
+be prepared offline, and handles exception words like "read" (present vs. past tense) correctly.
+Other languages can be extended simply by providing a lexicon JSON file.
+
+### Neural (Protophone)
+
+For words not found in the lexicon, a neural phonemizer takes over. **Protophone** is a pure
+C++ CTC-based model with SIMD-accelerated inference via xsimd (AVX2+FMA on x86_64, NEON on ARM).
+When built with `ET_ON=ON`, inference is delegated to **ExecuTorch** instead, enabling optimized
+on-device acceleration.
+
+These strategies are combined through a **hybrid phonemizer**: the lexicon is tried first, and
+the neural model serves as a fallback. Each level can be individually enabled or disabled via
+configuration.
+
+### Extending to a New Language
+
+Phonemis is designed to be contribution-friendly. Adding support for a new language requires
+three files under `src/phonemis/lang/XX/`:
+
+| File | Purpose | Requirement |
+|------|---------|-------------|
+| `constants.h` | Language-specific data: number words (cardinals, ordinals), currencies, units, months, symbol words, tokenizer special-character rules. | Mandatory |
+| `num2word.h` / `num2word.cpp` | Number verbalizer extending `processor::num2word::Num2WordLayer<NumberReading>`: the language's number notation, its own patterns (dates, phones, ordinal suffixes), how context decides a reading (`resolve`) and the readings themselves (`read_*`). | Mandatory |
+| `patterns.h` / `patterns.cpp` | Number patterns the generic ones do not cover (Roman ordinals, law numbers). | Optional |
+| `pipeline.h` | Pipeline class extending `IPipeline`. Orchestrates preprocessing layers (trimming, number and symbol verbalization), a tokenizer, and a `HybridPhonemizer<LexiconPhonemizer, NeuralPhonemizer>`. | Mandatory |
+| Lexicon JSON / Protophone model | Word-to-phoneme mappings **or** trained neural model (`.bin`). | At least one required |
+
+Then register the language in the factory method at `src/phonemis/base/pipeline.cpp`.
+
+## License
+
+The source code of Phonemis is released under the [MIT license](LICENSE). Some of the data in `data/` and the bundled
+libraries in `src/third-party/` come from other projects under their own licenses: see
+[THIRD_PARTY_NOTICES.md](THIRD_PARTY_NOTICES.md). In particular, the English part-of-speech tagger (`tagger.json`)
+is derived from a corpus licensed for non-commercial use only, so leave out `Config::tagger` in commercial products.