react-native-executorch 0.10.6 → 0.10.7

package/third-party/common/phonemis/src/phonemis/lang/fr/patterns.cpp

@@ -0,0 +1,83 @@
+#include "patterns.h"
+#include <phonemis/base/processor/layers/num2word/primitives.h>
+#include <phonemis/utils/conversions.h>
+
+#include <array>
+#include <optional>
+#include <string>
+
+namespace phonemis::fr {
+
+using namespace processor;
+using namespace processor::num2word;
+
+namespace {
+int roman_digit(char32_t c) {
+  switch (c) {
+    case U'I':
+      return 1;
+    case U'V':
+      return 5;
+    case U'X':
+      return 10;
+    case U'L':
+      return 50;
+    case U'C':
+      return 100;
+    default:
+      return 0;
+  }
+}
+
+// The value of a Roman numeral written in its canonical form (XIX, not XVIIII).
+std::optional<int> roman_value(std::u32string_view numeral) {
+  int value = 0;
+  for (size_t i = 0; i < numeral.size(); ++i) {
+    int digit = roman_digit(numeral[i]);
+    value += i + 1 < numeral.size() && digit < roman_digit(numeral[i + 1]) ? -digit : digit;
+  }
+
+  static constexpr std::array<std::pair<int, std::u32string_view>, 9> kSymbols = {
+      {{100, U"C"}, {90, U"XC"}, {50, U"L"}, {40, U"XL"}, {10, U"X"}, {9, U"IX"}, {5, U"V"}, {4, U"IV"}, {1, U"I"}}};
+  std::u32string canonical;
+  for (int rest = value; const auto& [symbol_value, symbol] : kSymbols) {
+    for (; rest >= symbol_value; rest -= symbol_value) {
+      canonical += symbol;
+    }
+  }
+  return canonical == numeral ? std::optional(value) : std::nullopt;
+}
+} // namespace
+
+std::optional<Match> RomanOrdinalPattern::match(std::u32string_view text, size_t position) const {
+  if (!is_clear_before(text, position)) {
+    return std::nullopt;
+  }
+
+  // Two Roman digits or more, or a single I (Ier)
+  size_t end = position;
+  while (end < text.size() && roman_digit(text[end])) {
+    end++;
+  }
+  std::u32string_view numeral = text.substr(position, end - position);
+  if (numeral.size() < 2 && numeral != U"I") {
+    return std::nullopt;
+  }
+  auto value = roman_value(numeral);
+  if (!value) {
+    return std::nullopt;
+  }
+
+  for (auto suffix : keys_at(text, end, {U"er", U"re", U"ère", U"e", U"ème"})) {
+    if (is_clear_after(text, end + suffix.size())) {
+      std::u32string digits = utils::conversions::utf8_to_u32(std::to_string(*value));
+      return Match{.kind = Kind::ORDINAL,
+                   .span = {position, end + suffix.size()},
+                   .numbers = {{.span = {position, end}, .integer = digits}},
+                   .suffix = suffix};
+    }
+  }
+  return std::nullopt;
+}
+
+} // namespace phonemis::fr

package/third-party/common/phonemis/src/phonemis/lang/fr/patterns.h

@@ -0,0 +1,15 @@
+#pragma once
+
+#include <phonemis/base/processor/layers/num2word/pattern.h>
+
+namespace phonemis::fr {
+
+// XXe, XIXème, Ier: an ordinal written in Roman numerals. Its number is the value of the numeral.
+class RomanOrdinalPattern : public processor::num2word::Pattern {
+public:
+  RomanOrdinalPattern() : Pattern(processor::num2word::Kind::ORDINAL) {}
+
+  std::optional<processor::num2word::Match> match(std::u32string_view text, size_t position) const override;
+};
+
+} // namespace phonemis::fr

package/third-party/common/phonemis/src/phonemis/lang/fr/pipeline.h

@@ -7,8 +7,8 @@
 #include <phonemis/base/ipipeline.h>
 #include <phonemis/base/phonemizer/hybrid_phonemizer.h>
 #include <phonemis/base/phonemizer/lexicon_phonemizer.h>
+#include <phonemis/base/processor/preprocessor.h>
 #include <phonemis/base/processor/processor.h>
-#include <phonemis/base/processor/trim_layer.h>
 #include <phonemis/base/tokenizer/tokenizer.h>
 #include <phonemis/utils/conversions.h>
 
@@ -24,8 +24,8 @@ public:
         phonemizer_(config.phonemizer) {
 
     // 1. Setup Preprocessing layers
-    preprocessor_.add_layer(std::make_unique<processor::TrimLayer>());
-    preprocessor_.add_layer(std::make_unique<Num2Word>());
+    preprocessor_ = processor::make_preprocessor<Num2Word>(
+        config.preprocessor, {.terms = &constants::term2word::kTerms, .symbols = &constants::sym2word::kSymbols});
   }
 
   // Performs a preprocessing stage of the pipeline.

package/third-party/common/phonemis/src/phonemis/lang/hi/constants.h

@@ -1,5 +1,9 @@
 #pragma once
 
+#include <phonemis/base/processor/common/conditions.h>
+#include <phonemis/base/processor/layers/normalize/types.h>
+#include <phonemis/base/processor/layers/num2word/notation.h>
+#include <phonemis/base/processor/layers/term2word/types.h>
 #include <phonemis/base/tokenizer/types.h>
 
 #include <array>
@@ -12,8 +16,25 @@
 
 namespace phonemis::hi::constants {
 
+// --- Character Normalization ---
+namespace normalize {
+  using ::phonemis::processor::normalize::Characters;
+
+  // Echo's classes, but the zero-width non-joiner and joiner stay: they decide how Devanagari letters join
+  inline const Characters kCharacters = {
+    {U"…",                U"..."}, // Ellipsis
+    {U"«»„“”‟˝",          U"\""},  // Quotes
+    {U"‘’‚‛´`ʼ",          U"'"},   // Apostrophes
+    {U"\u200b\ufeff",     U""}     // Invisible: zero-width space, byte order mark
+  };
+} // namespace normalize
+
 // --- Number to Word Normalization ---
 namespace num2word {
+  using ::phonemis::processor::num2word::DateFormat;
+  using ::phonemis::processor::num2word::Joiners;
+  using ::phonemis::processor::num2word::keys_of;
+  using ::phonemis::processor::num2word::Notation;
   // Hindi has unique cardinal words for every number 0-99 — they are not
   // compositionally derivable from tens + units like in most European
   // languages. Above 99 the construction becomes regular: hundreds use
@@ -53,10 +74,10 @@ namespace num2word {
 
   // Currencies. Hindi loanword currencies are invariable in form, so only the
   // singular noun is stored.
-  inline const std::unordered_map<char32_t, std::u32string> kCurrencies = {
-    {U'$', U"डॉलर"},
-    {U'€', U"यूरो"},
-    {U'£', U"पाउंड"}
+  inline const std::unordered_map<std::u32string, std::u32string> kCurrencies = {
+    {U"$", U"डॉलर"},
+    {U"€", U"यूरो"},
+    {U"£", U"पाउंड"}
   };
 
   // Months: standard Gregorian month names in Hindi (used as-is in dates).
@@ -84,15 +105,46 @@ namespace num2word {
     U"वाँ", U"वीं", U"वें"
   };
 
-  // Recognized ordinal suffixes for digit-attached notation. Only the
-  // alpha-only base "व" is matched at scan time; trailing matras / nasal
-  // marks of the suffix are reattached to the verbalized form because the
-  // base scanner does not classify combining marks as alphabetic.
-  inline const std::unordered_set<std::u32string> kOrdinalSuffixes = {
-    U"व"
+  // Ordinal suffixes after digits, in the order of OrdinalForms: 5वाँ, 5वीं, 5वें
+  inline const std::vector<std::u32string> kOrdinalSuffixes = {
+    U"वाँ", U"वीं", U"वें"
+  };
+
+  // --- Notation: how numbers are written ---
+  inline const Notation kNotation = {
+    .group_separators = U",", .decimal_separators = U".", .time_separators = U":", .currencies = keys_of(kCurrencies),
+    .fractions = true
+  };
+
+  // Words between numbers: 3.8.10, 10.0.0.51, 10-15, 1920x1080, 2:1, -5, 2 + 2
+  inline const Joiners kJoiners = {
+    .point = U"दशमलव", .dot = U"दशमलव", .range = U" से ", .by = U"गुणा", .score = U" ", .minus = U"माइनस", .plus = U"प्लस"
   };
+
+  // 27.03.2026, 27-03-2026: the year has four digits
+  inline const DateFormat kDateFormat = {.separators = U".-/", .short_year = false};
 } // namespace num2word
 
+// --- Symbols to Word Normalization ---
+namespace sym2word {
+  using ::phonemis::processor::Readings;
+
+  // TODO: not verified against a reference verbalizer yet
+  inline const Readings kSymbols = {
+    {U"&", {U"और"}},       {U"+", {U"प्लस"}},     {U"=", {U"बराबर"}},    {U"%", {U"प्रतिशत"}},  {U"@", {U"एट"}},
+    {U"°", {U"डिग्री"}},    {U"$", {U"डॉलर"}},     {U"€", {U"यूरो"}},      {U"£", {U"पाउंड"}},    {U"₹", {U"रुपये"}}
+  };
+} // namespace sym2word
+
+// --- Terms to Word Normalization ---
+namespace term2word {
+  using ::phonemis::processor::term2word::Terms;
+
+  inline const Terms kTerms = {
+    .web = {.dot = U"डॉट", .at = U"एट", .slash = U"स्लैश"}
+  };
+} // namespace term2word
+
 // --- Tokenization Rules ---
 namespace tokenizer {
   using ::phonemis::tokenizer::split::Rule;

package/third-party/common/phonemis/src/phonemis/lang/hi/num2word.cpp

@@ -1,162 +1,113 @@
-#include "num2word.h"
 #include "constants.h"
-#include <phonemis/utils/conversions.h>
+#include "num2word.h"
+#include <phonemis/base/processor/layers/num2word/numerals.h>
+#include <phonemis/base/processor/layers/num2word/patterns.h>
+
 #include <algorithm>
-#include <cmath>
 #include <cstddef>
-#include <cstdlib>
 
 namespace phonemis::hi {
 
-using namespace constants;
-using namespace phonemis::utils;
+using namespace constants::num2word;
+using namespace processor;
+using namespace processor::num2word;
 
 namespace {
-
-// Hindi ordinals split into three gender/number forms. The numeric values
-// align with the column order used in `kOrdinalsSpecial` and `kOrdinalEndings`.
-enum class OrdinalForm : std::size_t { MASCULINE = 0, FEMININE = 1, PLURAL = 2 };
-
-// Maps an ordinal suffix to a gender/number form. The base scanner only
-// captures the alpha portion ("व") of a Devanagari suffix because matras and
-// nasal marks aren't classified as alphabetic, so the masculine form is the
-// only one selectable through digit-attached notation in raw text. Direct
-// API callers can pass the full ending ("वाँ", "वीं", "वें") to pick a
-// specific form.
-OrdinalForm form_from_suffix(std::u32string_view suffix) {
-  if (suffix == U"वीं") return OrdinalForm::FEMININE;
-  if (suffix == U"वें") return OrdinalForm::PLURAL;
-  return OrdinalForm::MASCULINE;
+uint64_t value(const Number& number) {
+  return value_of(number.integer).value_or(0);
 }
 
-} // namespace
-
-std::u32string Num2Word::to_cardinal_int(int32_t value) const {
-  if (value < 0) {
-    return U"माइनस " + to_cardinal_int(std::abs(value));
+// Every number up to 99 is its own word; then "<digit> सौ", and the Indian scales (हज़ार, लाख, करोड़)
+std::u32string cardinal(uint64_t n) {
+  if (n < 100) {
+    return kCardinals.at(static_cast<int32_t>(n));
   }
-
-  if (auto it = num2word::kCardinals.find(value); it != num2word::kCardinals.end()) {
-    return it->second;
+  if (n < 1000) {
+    uint64_t rest = n % 100;
+    return kCardinals.at(static_cast<int32_t>(n / 100)) + U" सौ" + (rest ? U" " + cardinal(rest) : U"");
   }
 
-  if (value < 1000) {
-    // 100-999: "<digit> सौ" with an optional remainder. Hindi keeps "एक"
-    // before "सौ" (one hundred reads as "एक सौ", not bare "सौ").
-    int32_t hundreds_high = value / 100;
-    int32_t rest = value % 100;
-    std::u32string head = num2word::kCardinals.at(hundreds_high) + U" सौ";
-    if (rest == 0) return head;
-    return head + U" " + to_cardinal_int(rest);
-  }
-
-  // Indian-numbering decomposition (हज़ार / लाख / करोड़) in descending order.
-  // Unlike Western numbering, the next scale up after हज़ार multiplies by 100
-  // (lakh = 100k) and then 100 again (crore = 10M), so 1,000,000 reads as
-  // "दस लाख" rather than as a million-scoped phrase.
-  for (auto it = num2word::kLargeCardinals.rbegin(); it != num2word::kLargeCardinals.rend(); ++it) {
-    int64_t base = it->first;
-    if (static_cast<int64_t>(value) >= base) {
-      int32_t high = value / static_cast<int32_t>(base);
-      int32_t low = value % static_cast<int32_t>(base);
-
-      // Hindi keeps the leading "एक" in front of every scale noun (e.g.
-      // 1000 -> "एक हज़ार", 100000 -> "एक लाख"), unlike European languages
-      // that drop the multiplier of one.
-      std::u32string res = to_cardinal_int(high) + U" " + it->second;
-      if (low > 0) {
-        res += U" " + to_cardinal_int(low);
-      }
-      return res;
+  // Hindi keeps the "एक" of every scale: एक हज़ार, एक लाख
+  for (auto it = kLargeCardinals.rbegin(); it != kLargeCardinals.rend(); ++it) {
+    auto base = static_cast<uint64_t>(it->first);
+    if (n >= base) {
+      uint64_t low = n % base;
+      return cardinal(n / base) + U" " + it->second + (low ? U" " + cardinal(low) : U"");
     }
   }
-
-  return conversions::utf8_to_u32(std::to_string(value));
+  return U"";
 }
 
-std::u32string Num2Word::to_cardinal_float(float value, std::u32string_view repr) const {
-  if (value < 0.0f) {
-    return U"माइनस " + to_cardinal_float(std::abs(value), repr);
-  }
-
-  std::string value_str = conversions::u32_to_utf8(repr);
-
-  // Trim trailing zeros and any dangling decimal point.
-  auto rit = std::find_if(value_str.rbegin(), value_str.rend(),
-                          [](char c) { return c != '0'; });
-  value_str = std::string(value_str.begin(), rit.base());
-  if (!value_str.empty() && value_str.back() == '.') {
-    value_str.pop_back();
-  }
-
-  size_t dot_pos = value_str.find('.');
-  if (dot_pos == std::string::npos) {
-    return to_cardinal_int(value_str.empty() ? 0 : std::stoi(value_str));
+std::u32string digits(std::u32string_view text) {
+  std::vector<std::u32string> words;
+  for (char32_t c : text) {
+    words.push_back(cardinal(c - U'0'));
   }
-
-  std::string integer_part_str = value_str.substr(0, dot_pos);
-  std::string fractional_part_str = value_str.substr(dot_pos + 1);
-  int32_t integer_part = integer_part_str.empty() ? 0 : std::stoi(integer_part_str);
-
-  // Hindi reads the fractional part digit-by-digit (e.g. 0.05 -> "शून्य
-  // दशमलव शून्य पाँच"), unlike languages that verbalize it as a whole number.
-  std::u32string frac_words;
-  for (char c : fractional_part_str) {
-    if (!frac_words.empty()) frac_words += U" ";
-    frac_words += to_cardinal_int(c - '0');
-  }
-
-  return to_cardinal_int(integer_part) + U" दशमलव " + frac_words;
+  return join(words);
 }
+} // namespace
 
-std::u32string Num2Word::to_ordinal_int(int32_t value, std::u32string_view suffix) const {
-  OrdinalForm form = form_from_suffix(suffix);
+Num2Word::Num2Word() : Num2WordLayer(kNotation, kJoiners) {
+  add_pattern(std::make_unique<DatePattern>(kDateFormat));
+  add_pattern(std::make_unique<SuffixPattern>(Kind::ORDINAL, kOrdinalSuffixes));
+}
 
-  if (auto it = num2word::kOrdinalsSpecial.find(value); it != num2word::kOrdinalsSpecial.end()) {
-    return it->second[static_cast<std::size_t>(form)];
+std::u32string Num2Word::read_number(const Number& number, const NumberReading& reading) const {
+  if (number.fraction.empty() && spoken_digits(number, 7)) {
+    return digits(number.integer);
   }
 
-  // Regular ordinal: cardinal phrase with the gender-specific ending fused
-  // onto the last token (no separating space, e.g. "इक्कीसवाँ").
-  return to_cardinal_int(value) + num2word::kOrdinalEndings[static_cast<std::size_t>(form)];
+  // The fraction is read digit by digit, without its trailing zeros: 0.50 -> शून्य दशमलव पाँच
+  std::u32string_view fraction = number.fraction;
+  fraction = fraction.substr(0, fraction.find_last_not_of(U'0') + 1);
+  std::u32string integer = cardinal(value(number));
+  return fraction.empty() ? integer : integer + U" दशमलव " + digits(fraction);
 }
 
-std::u32string Num2Word::to_currency(char32_t currency, std::variant<int32_t, float> /*number*/) const {
-  // Hindi loanword currencies (डॉलर, यूरो, पाउंड) are invariant; the same
-  // form is used regardless of the amount.
-  if (auto it = num2word::kCurrencies.find(currency); it != num2word::kCurrencies.end()) {
-    return it->second;
+std::u32string Num2Word::read_code_digits(std::u32string_view text) const {
+  auto n = value_of(text);
+  if ((text.size() > 1 && text[0] == U'0') || text.size() >= 7 || !n) {
+    return digits(text);
   }
-  return std::u32string(1, currency);
+  return cardinal(*n);
 }
 
-std::u32string Num2Word::to_fraction(int32_t numerator, int32_t denominator) const {
-  // Common fractions have dedicated Hindi nouns; everything else falls back
-  // to the colloquial "<num> बटा <den>" pattern.
-  if (numerator == 1 && denominator == 2) return U"आधा";
-  if (denominator == 3) return to_cardinal_int(numerator) + U" तिहाई";
-  if (denominator == 4) return to_cardinal_int(numerator) + U" चौथाई";
-  return to_cardinal_int(numerator) + U" बटा " + to_cardinal_int(denominator);
+std::u32string Num2Word::read_date(const Match& match, const NumberReading& reading) const {
+  // सत्ताईस मार्च दो हज़ार छब्बीस
+  return cardinal(value(match.numbers[0])) + U" " + kMonths[value(match.numbers[1])] + U" " +
+         cardinal(value(match.numbers[2]));
 }
 
-std::u32string Num2Word::to_day(uint32_t day) const {
-  // Hindi dates render the day as a plain cardinal (e.g. "27 मार्च").
-  return to_cardinal_int(static_cast<int32_t>(day));
+std::u32string Num2Word::read_fraction(const Match& match, const NumberReading& reading) const {
+  // Common fractions have their own nouns, others are "<a> बटा <b>"
+  uint64_t a = value(match.numbers[0]), b = value(match.numbers[1]);
+  if (a == 1 && b == 2) {
+    return U"आधा";
+  }
+  if (b == 3 || b == 4) {
+    return cardinal(a) + (b == 3 ? U" तिहाई" : U" चौथाई");
+  }
+  return cardinal(a) + U" बटा " + cardinal(b);
 }
 
-std::u32string Num2Word::to_month(uint32_t month) const {
-  if (month >= 1 && month <= 12) {
-    return num2word::kMonths[month];
+std::u32string Num2Word::read_ordinal(const Match& match, const NumberReading& reading) const {
+  // The suffix gives the form: masculine (वाँ), feminine (वीं) or plural (वें)
+  size_t form = std::ranges::find(kOrdinalSuffixes, match.suffix) - kOrdinalSuffixes.begin();
+  uint64_t n = value(match.numbers[0]);
+
+  // पहला, दूसरा, तीसरा, चौथा, छठा: irregular
+  if (auto it = kOrdinalsSpecial.find(static_cast<int32_t>(n)); n < 10 && it != kOrdinalsSpecial.end()) {
+    return it->second[form];
   }
-  return U"";
+  return cardinal(n) + kOrdinalEndings[form];
 }
 
-std::u32string Num2Word::to_year(uint32_t year) const {
-  return to_cardinal_int(static_cast<int32_t>(year));
+std::u32string Num2Word::read_money(const Number& number, const Match& match, const NumberReading& reading) const {
+  return read_number(number, reading) + U" " + kCurrencies.at(std::u32string(match.unit));
 }
 
-bool Num2Word::is_ordinal_suffix(std::u32string_view suffix) const {
-  return num2word::kOrdinalSuffixes.contains(std::u32string(suffix));
+std::u32string Num2Word::read_percent(const Number& number, const Match& match, const NumberReading& reading) const {
+  return read_number(number, reading) + U" प्रतिशत";
 }
 
-} // namespace phonemis::hi
+} // namespace phonemis::hi

package/third-party/common/phonemis/src/phonemis/lang/hi/num2word.h

@@ -1,25 +1,36 @@
 #pragma once
 
-#include <phonemis/base/processor/num2word/layer.h>
+#include <phonemis/base/processor/layers/num2word/layer.h>
+
 #include <string>
-#include <variant>
 
 namespace phonemis::hi {
 
-class Num2Word : public processor::num2word::Num2WordLayer {
+// Hindi numbers are read without context so far.
+struct NumberReading {};
+
+// TODO: outdated - Hindi readings predate the context-aware design and have no reference verbalizer.
+// Missing: Indian digit grouping (1,00,000), Devanagari digits, units and context (gender) of the counted noun.
+class Num2Word : public processor::num2word::Num2WordLayer<NumberReading> {
 public:
-    using Num2WordLayer::Num2WordLayer;
-
-    char32_t decimal_separator() const override { return U'.'; }
-    std::u32string to_cardinal_int(int32_t number) const override;
-    std::u32string to_cardinal_float(float number, std::u32string_view repr) const override;
-    std::u32string to_ordinal_int(int32_t number, std::u32string_view suffix = U"") const override;
-    std::u32string to_currency(char32_t currency, std::variant<int32_t, float> number) const override;
-    std::u32string to_fraction(int32_t numerator, int32_t denominator) const override;
-    std::u32string to_day(uint32_t day) const override;
-    std::u32string to_month(uint32_t month) const override;
-    std::u32string to_year(uint32_t year) const override;
-    bool is_ordinal_suffix(std::u32string_view suffix) const override;
+  Num2Word();
+
+protected:
+  using Context = processor::Context;
+  using Match = processor::num2word::Match;
+  using Number = processor::num2word::Number;
+
+  // Resolution
+  NumberReading resolve(const Match& match, const Context& context) const override { return {}; }
+
+  // Readings
+  std::u32string read_number(const Number& number, const NumberReading& reading) const override;
+  std::u32string read_code_digits(std::u32string_view digits) const override;
+  std::u32string read_date(const Match& match, const NumberReading& reading) const override;
+  std::u32string read_fraction(const Match& match, const NumberReading& reading) const override;
+  std::u32string read_ordinal(const Match& match, const NumberReading& reading) const override;
+  std::u32string read_money(const Number& number, const Match& match, const NumberReading& reading) const override;
+  std::u32string read_percent(const Number& number, const Match& match, const NumberReading& reading) const override;
 };
 
-} // namespace phonemis::hi
+} // namespace phonemis::hi

package/third-party/common/phonemis/src/phonemis/lang/hi/pipeline.h

@@ -8,8 +8,8 @@
 #include <phonemis/base/ipipeline.h>
 #include <phonemis/base/phonemizer/hybrid_phonemizer.h>
 #include <phonemis/base/phonemizer/lexicon_phonemizer.h>
+#include <phonemis/base/processor/preprocessor.h>
 #include <phonemis/base/processor/processor.h>
-#include <phonemis/base/processor/trim_layer.h>
 #include <phonemis/base/tokenizer/tokenizer.h>
 
 #include <memory>
@@ -24,8 +24,10 @@ public:
         phonemizer_(make_phonemizer_config(config.phonemizer)) {
 
     // 1. Setup Preprocessing layers
-    preprocessor_.add_layer(std::make_unique<processor::TrimLayer>());
-    preprocessor_.add_layer(std::make_unique<Num2Word>());
+    preprocessor_ =
+        processor::make_preprocessor<Num2Word>(config.preprocessor, {.characters = &constants::normalize::kCharacters,
+                                                                     .terms = &constants::term2word::kTerms,
+                                                                     .symbols = &constants::sym2word::kSymbols});
   }
 
   // Performs a preprocessing stage of the pipeline.