react-native-executorch 0.10.5 → 0.10.7

package/third-party/common/phonemis/src/phonemis/lang/en/num2word.cpp

@@ -1,194 +1,291 @@
-#include "num2word.h"
 #include "constants.h"
-#include <phonemis/base/processor/constants.h>
-#include <phonemis/utils/conversions.h>
+#include "num2word.h"
+#include <phonemis/base/processor/layers/num2word/numerals.h>
+#include <phonemis/base/processor/layers/num2word/patterns.h>
 #include <phonemis/utils/strings.h>
-#include <algorithm>
-#include <cmath>
-#include <sstream>
-#include <vector>
-#include <iterator>
+#include <phonemis/utils/unicode.h>
 
 namespace phonemis::en {
 
-using namespace constants;
-using namespace phonemis::processor;
+using namespace constants::num2word;
+using namespace processor;
+using namespace processor::num2word;
 using namespace utils;
 
-std::u32string Num2Word::to_cardinal_int(int32_t value) const {
-  if (value < 0) {
-    return U"minus " + to_cardinal_int(std::abs(value));
-  }
+namespace {
+// Number words
 
-  if (num2word::kCardinals.contains(static_cast<int>(value))) {
-    return num2word::kCardinals.at(static_cast<int>(value));
+std::u32string cardinal(uint64_t n) {
+  if (n < 20) {
+    return std::u32string(kOnes[n]);
   }
-
-  if (value < 100) {
-    int32_t tens = value / 10;
-    int32_t units = value % 10;
-    return num2word::kCardinals.at(tens * 10) + U" " + num2word::kCardinals.at(units);
+  if (n < 100) {
+    return std::u32string(kTens[n / 10]) + (n % 10 ? U"-" + std::u32string(kOnes[n % 10]) : U"");
+  }
+  if (n < 1000) {
+    return std::u32string(kOnes[n / 100]) + U" hundred" + (n % 100 ? U" " + cardinal(n % 100) : U"");
+  }
+  for (const auto& [value, word] : kScaleWords) {
+    if (n >= value) {
+      return cardinal(n / value) + U" " + word + (n % value ? U" " + cardinal(n % value) : U"");
+    }
   }
+  return U"";
+}
 
-  // Handle large numbers by iterating through the large scales map (hundred, thousand, etc.)
-  // in descending order to find the largest applicable base.
-  for (auto it = num2word::kLargeCardinals.rbegin(); it != num2word::kLargeCardinals.rend(); ++it) {
-    int64_t base = it->first;
-    if (static_cast<int64_t>(value) >= base) {
-      int32_t high = value / static_cast<int32_t>(base);
-      int32_t low = value % static_cast<int32_t>(base);
+std::u32string ordinal(uint64_t n) {
+  // Only the last word changes: twenty-first, one hundred and twelfth
+  std::u32string words = cardinal(n);
+  size_t start = words.find_last_of(U" -") + 1;
+  std::u32string last = words.substr(start);
+  words.erase(start);
 
-      std::u32string res = to_cardinal_int(high) + U" " + it->second;
-      if (low > 0) {
-        // Use "and" for values below 100 as per English counting convention (e.g., "one hundred and five")
-        std::u32string sep = (low < 100) ? U" and " : U", ";
-        res += sep + to_cardinal_int(low);
-      }
+  if (auto it = kIrregularOrdinals.find(last); it != kIrregularOrdinals.end()) {
+    return words + it->second;
+  }
+  return words + (last.ends_with(U'y') ? last.substr(0, last.size() - 1) + U"ieth" : last + U"th");
+}
 
-      return res;
-    }
+// Year-style reading, also of 3-4 digit codes: nineteen oh five, two thousand seven, nine fifty-six
+std::u32string pairs(uint64_t n) {
+  uint64_t high = n / 100, low = n % 100;
+  if (n >= 1000 && n % 1000 < 10) {
+    return cardinal(n);
+  }
+  if (low == 0) {
+    return cardinal(high) + U" hundred";
   }
+  return cardinal(high) + U" " + (low < 10 ? U"oh " + std::u32string(kOnes[low]) : cardinal(low));
+}
 
-  // Fallback - return the number without verbalization.
-  return conversions::utf8_to_u32(std::to_string(value));
+std::u32string digits(std::u32string_view text) {
+  std::vector<std::u32string> words;
+  for (char32_t c : text) {
+    words.emplace_back(kOnes[c - U'0']);
+  }
+  return join(words);
 }
 
-std::u32string Num2Word::to_cardinal_float(float value, std::u32string_view repr) const {
-    if (value < 0.0f) {
-        return U"minus " + to_cardinal_float(std::abs(value), repr);
-    }
+// a.m. and p.m. as their lowercase abbreviations are said; AM and PM stay as written
+std::u32string ampm(std::u32string_view suffix) {
+  std::u32string base = strings::to_lower(suffix.substr(0, suffix.find_last_not_of(U'.') + 1));
+  bool lowercase = std::ranges::none_of(suffix, [](char32_t c) { return unicode::isupper(c); });
+  if (lowercase && (base == U"am" || base == U"pm")) {
+    return base == U"am" ? U"a.m." : U"p.m.";
+  }
+  return std::u32string(suffix);
+}
 
-    // Find dot (brak point) and split the float into 2 parts: integer part and fractional part.
-    // We will handle both parts separately and concatenate the results.
-    std::string tmp_str = conversions::u32_to_utf8(repr);
-    auto rit = std::find_if(tmp_str.rbegin(), tmp_str.rend(),
-                            [](char c) -> bool { return c != '0'; });
-    std::string value_str = std::string(tmp_str.begin(), rit.base());
+uint64_t value(const Number& number) {
+  return value_of(number.integer).value_or(0);
+}
+} // namespace
 
-    // Handle cases such as 5. - where float can be represented directly as an integer
-    if (!value_str.empty() && value_str.back() == '.') {
-      value_str.pop_back();
-    }
+Num2Word::Num2Word() : Num2WordLayer(kNotation, kJoiners) {
+  add_pattern(std::make_unique<DatePattern>(kDateFormat));
+  for (const auto& format : kPhoneFormats) {
+    add_pattern(std::make_unique<PhonePattern>(format));
+  }
+  add_pattern(std::make_unique<SuffixPattern>(Kind::ORDINAL, kOrdinalSuffixes));
+  add_pattern(std::make_unique<SuffixPattern>(Kind::DECADE, kDecadeSuffixes, kDecadeFormat));
+}
 
-    size_t dot_pos = value_str.find('.');
-    if (dot_pos == std::string::npos) {
-      return to_cardinal_int(std::stoi(value_str));
+// -----------------------
+// ----- Resolution -----
+// -----------------------
+
+bool Num2Word::accept(const Match& match, const Context& context) const {
+  switch (match.kind) {
+    case Kind::FRACTION:
+      return kFractions.contains({value(match.numbers[0]), value(match.numbers[1])});
+    case Kind::SCALE:
+      return match.scale != U"K"; // 4K: a resolution
+    case Kind::TIME: {
+      uint64_t hours = value(match.numbers[0]);
+      // 7pm: an hour alone needs am or pm
+      if (match.numbers.size() == 1) {
+        return hours <= 12 && strings::contains(std::u32string_view(U"apAP"), match.suffix[0]);
+      }
+      // 1.13 is a decimal, unless a time is expected: at 10.30
+      bool decimal = match.separator == U'.' && match.suffix.empty() && !kTimeWords.contains(context.before());
+      return hours <= 24 && value(match.numbers[1]) <= 59 && !decimal;
     }
+    default:
+      return true;
+  }
+}
 
-    std::string integer_part_str = value_str.substr(0, dot_pos);
-    std::string fractional_part_str = value_str.substr(dot_pos + 1);
-
-    // Standard, cardinal verbalization of the integer part
-    int32_t integer_part = integer_part_str.empty() ? 0 : std::stoi(integer_part_str);
-    std::u32string res = to_cardinal_int(integer_part) + U" point";
+NumberReading Num2Word::resolve(const Match& match, const Context& context) const {
+  std::u32string previous = context.before();
+  std::u32string next = context.after();
+  std::u32string next_lower = strings::to_lower(next);
 
-    // Digit-by-digit verbalization of the fractional part.
-    for (char c : fractional_part_str) {
-      if (isdigit(static_cast<unsigned char>(c))) {
-        res += U" " + to_cardinal_int(c - '0');
-      }
-    }
+  // 1999 people: a plural noun after a 4-digit number makes it a count, unless a year is expected (in 1999)
+  bool plural_noun =
+      kIrregularPlurals.contains(next_lower) || (next_lower.size() > 3 && next_lower.ends_with(U's') &&
+                                                 !next_lower.ends_with(U"ss") && !kStopWords.contains(next_lower));
+  bool year_cue = kYearCues.contains(previous) || kMonths.contains(previous);
 
-    return res;
+  return {.after_month = kMonths.contains(previous),
+          .counted = plural_noun && !year_cue,
+          .attributive = !next.empty() && unicode::isalpha(next[0]) && !kStopWords.contains(next_lower)};
 }
 
-std::u32string Num2Word::to_ordinal_int(int32_t value, std::u32string_view suffix) const {
-  // A helper function to create a proper suffix based on verbalized representation.
-  auto get_ordinal_suffix_word = [](const std::u32string& word) -> std::u32string {
-    return num2word::kOrdinals.contains(word) ? num2word::kOrdinals.at(word) :
-           word.empty()                       ? U"th" :
-           word.back() == U'y'                ? word.substr(0, word.length() - 1) + U"ieth" :
-                                                word + U"th";
-  };
+// --------------------
+// ----- Readings -----
+// --------------------
 
-  std::u32string card = to_cardinal_int(value);
-  std::vector<std::u32string> words = strings::split(card, U' ');
-  
-  if (words.empty()) {
-    return card;
+std::u32string Num2Word::read_integer(const Number& number, const NumberReading& reading) const {
+  auto n = value_of(number.integer);
+  if (!n || spoken_digits(number, 5)) {
+    return digits(number.integer);
+  }
+  if (reading.after_month && *n >= 1 && *n <= 31) {
+    return ordinal(*n);
   }
+  if (number.integer.size() == 4 && !number.grouped) {
+    return reading.counted ? cardinal(*n) : pairs(*n);
+  }
+  return cardinal(*n);
+}
 
-  std::u32string& last = words.back();
-  size_t dash_pos = last.find(U'-');
-  if (dash_pos != std::u32string::npos) {
-    std::vector<std::u32string> parts = strings::split(last, U'-');
-    if (!parts.empty()) {
-      parts.back() = get_ordinal_suffix_word(parts.back());
-      last = parts.size() > 1 ? parts[0] + U"-" + parts[1] : parts[0];
-    }
-  } else {
-    last = get_ordinal_suffix_word(last);
+std::u32string Num2Word::read_number(const Number& number, const NumberReading& reading) const {
+  if (number.fraction.empty()) {
+    return read_integer(number, reading);
   }
 
-  std::u32string res = words[0];
-  for (size_t i = 1; i < words.size(); ++i) {
-    res += U" " + words[i];
+  // .5 has no zero before its point
+  std::u32string head = number.integer.empty() ? U"" : cardinal(value(number)) + U" ";
+  return head + (number.fraction == U"0" ? U"point oh" : U"point " + digits(number.fraction));
+}
+
+std::u32string Num2Word::read_code_digits(std::u32string_view text) const {
+  auto n = value_of(text);
+  if (text.size() > 1 && text[0] == U'0') {
+    std::vector<std::u32string> words;
+    for (char32_t c : text) {
+      words.emplace_back(c == U'0' ? U"oh" : kOnes[c - U'0']);
+    }
+    return join(words);
+  }
+  if (!n || text.size() > 4) {
+    return digits(text);
   }
-  return res;
+  return text.size() <= 2 ? cardinal(*n) : pairs(*n);
 }
 
-std::u32string Num2Word::to_currency(char32_t currency, std::variant<int32_t, float> number) const {
-  if (!num2word::kCurrencies.contains(currency)) {
-    return std::u32string(1, currency);
+std::u32string Num2Word::read_date(const Match& match, const NumberReading& reading) const {
+  std::u32string words = ordinal(value(match.numbers[0])) + U" " + kMonthNames[value(match.numbers[1]) - 1];
+  if (match.numbers.size() < 3) {
+    return words;
   }
+  const Number& year = match.numbers[2];
+  return words + U" " + (year.integer.size() == 4 ? pairs(value(year)) : cardinal(value(year)));
+}
 
-  std::u32string result = num2word::kCurrencies.at(currency);
-  
-  // Determine if the amount is singular (exactly 1) to decide on pluralization.
-  // For floats, we use a small epsilon for comparison.
-  bool is_singular = std::visit([](auto&& arg) -> bool {
-    using T = std::decay_t<decltype(arg)>;
-    if constexpr (std::is_same_v<T, int32_t>) {
-      return arg == 1;
-    } else {
-      return std::abs(arg - 1.0F) < 1e-6F;
-    }
-  }, number);
+std::u32string Num2Word::read_phone(const Match& match, const NumberReading& reading) const {
+  // 1-555-...: the leading one joins the area code
+  std::vector<std::u32string> groups;
+  for (const auto& number : match.numbers) {
+    groups.push_back(number.integer);
+  }
+  if (match.text.starts_with(U"1-")) {
+    groups[0] = U"1" + groups[0];
+  }
 
-  if (!is_singular) {
-    result += U"s";
+  std::vector<std::u32string> words;
+  for (const auto& group : groups) {
+    words.push_back(digits(group));
   }
+  return join(words, U", ");
+}
 
-  return result;
+std::u32string Num2Word::read_fraction(const Match& match, const NumberReading& reading) const {
+  return kFractions.at({value(match.numbers[0]), value(match.numbers[1])});
 }
 
-std::u32string Num2Word::to_month(uint32_t month) const {
-  if (month >= 1 && month <= 12) {
-    return num2word::kMonths[month];
+std::u32string Num2Word::read_time(const Match& match, const NumberReading& reading) const {
+  std::u32string hours = cardinal(value(match.numbers[0]));
+  std::u32string suffix = ampm(match.suffix);
+  if (match.numbers.size() == 1) {
+    return hours + U" " + suffix;
   }
 
-  return U"";
+  // ten thirty, ten oh five, ten o'clock
+  const auto& minutes = match.numbers[1].integer;
+  std::u32string words = hours;
+  if (minutes != U"00") {
+    uint64_t m = value(match.numbers[1]);
+    words += minutes[0] == U'0' ? U" oh " + std::u32string(kOnes[m]) : U" " + cardinal(m);
+  }
+  if (!suffix.empty()) {
+    return words + U" " + suffix;
+  }
+  return words + (minutes == U"00" ? U" o'clock" : U"");
+}
+
+std::u32string Num2Word::read_ordinal(const Match& match, const NumberReading& reading) const {
+  return ordinal(value(match.numbers[0]));
+}
+
+std::u32string Num2Word::read_decade(const Match& match, const NumberReading& reading) const {
+  return *plural(read_number(match.numbers[0], reading));
 }
 
-std::u32string Num2Word::to_year(uint32_t year) const {
-  int32_t value = static_cast<int32_t>(year);
-  if (value == 0) return to_cardinal_int(0);
+std::u32string Num2Word::read_money(const Number& number, const Match& match, const NumberReading& reading) const {
+  const Forms& currency = kCurrencies.at(std::u32string(match.unit));
 
-  int32_t high = value / 100;
-  int32_t low = value % 100;
+  // A price: four ninety-nine, ninety-nine cents
+  if (number.fraction.size() == 2 && match.scale.empty()) {
+    uint64_t cents_value = value_of(number.fraction).value_or(0);
+    if (value(number) == 0) {
+      return cardinal(cents_value) + U" cents";
+    }
+    std::u32string cents =
+        number.fraction[0] == U'0' ? U"oh " + std::u32string(kOnes[cents_value]) : cardinal(cents_value);
+    Number whole{.integer = number.integer, .grouped = number.grouped};
+    return read_integer(whole, reading) + U" " + cents;
+  }
 
-  if (high == 0 || (high % 10 == 0 && low < 10) || high >= 100) {
-    return to_cardinal_int(value);
+  std::u32string words = read_number(number, reading);
+  if (!match.scale.empty()) {
+    words += U" " + kScales.at(std::u32string(match.scale));
   }
+  bool one = number.integer == U"1" && number.fraction.empty() && match.scale.empty();
+  return words + U" " + (one || reading.attributive ? currency.one : currency.many);
+}
 
-  std::u32string high_text = to_cardinal_int(high);
-  std::u32string low_text = low == 0 ? U"hundred" :
-                            low < 10 ? U"oh-" + to_cardinal_int(low) :
-                                       to_cardinal_int(low);
+std::u32string Num2Word::read_scale(const Number& number, const Match& match, const NumberReading& reading) const {
+  return read_number(number, reading) + U" " + kScales.at(std::u32string(match.scale));
+}
 
-  return high_text + U" " + low_text;
+std::u32string Num2Word::read_percent(const Number& number, const Match& match, const NumberReading& reading) const {
+  return read_number(number, reading) + U" percent" +
+         (match.suffix.empty() ? U"" : U" " + std::u32string(match.suffix));
 }
 
-bool Num2Word::is_ordinal_suffix(std::u32string_view suffix) const {
-    return suffix == U"st" || suffix == U"nd" || suffix == U"rd" || suffix == U"th";
+std::u32string Num2Word::read_unit(const Number& number, const Match& match, const NumberReading& reading) const {
+  const Forms& unit = kUnits.at(std::u32string(match.unit));
+  bool one = number.fraction.empty() && value(number) == 1;
+  return read_number(number, reading) + U" " + (one || reading.attributive ? unit.one : unit.many);
 }
 
-std::u32string Num2Word::to_fraction(int32_t numerator, int32_t denominator) const {
-    return to_cardinal_int(numerator) + U" " + to_ordinal_int(denominator);
+std::u32string Num2Word::read_range(const Number& from, const Number& to, const NumberReading& reading) const {
+  if (!from.fraction.empty() || !to.fraction.empty()) {
+    return joiners_.range;
+  }
+
+  // 10-15 is "ten to fifteen"; but scores and abbreviated seasons keep "-": 51-44, 2016-17, 21-22
+  uint64_t a = value(from), b = value(to);
+  bool season = from.integer.size() == 2 && b == a + 1;
+  return from.integer.size() == to.integer.size() && b > a && !season ? joiners_.range : U"-";
 }
 
-std::u32string Num2Word::to_day(uint32_t day) const {
-    return to_ordinal_int(static_cast<int32_t>(day));
+std::optional<std::u32string> Num2Word::plural(std::u32string_view words) const {
+  if (words.ends_with(U'y')) {
+    return std::u32string(words.substr(0, words.size() - 1)) + U"ies";
+  }
+  return std::u32string(words) + (words.ends_with(U'x') ? U"es" : U"s");
 }
 
 } // namespace phonemis::en

package/third-party/common/phonemis/src/phonemis/lang/en/num2word.h

@@ -1,26 +1,51 @@
 #pragma once
 
-#include <phonemis/base/processor/num2word/layer.h>
+#include <phonemis/base/processor/layers/num2word/layer.h>
+
+#include <optional>
 #include <string>
-#include <vector>
-#include <map>
 
 namespace phonemis::en {
 
-class Num2Word : public processor::num2word::Num2WordLayer {
+// How an English number is read, from the words around it.
+struct NumberReading {
+  bool after_month = false; // March 3: a day of the month, read as an ordinal
+  bool counted = false;     // 1999 people: a count rather than a year
+  bool attributive = false; // a $5 bill: before a noun, the unit stays singular
+};
+
+class Num2Word : public processor::num2word::Num2WordLayer<NumberReading> {
 public:
-    using Num2WordLayer::Num2WordLayer;
-
-    char32_t decimal_separator() const override { return U'.'; }
-    std::u32string to_cardinal_int(int32_t number) const override;
-    std::u32string to_cardinal_float(float number, std::u32string_view repr) const override;
-    std::u32string to_ordinal_int(int32_t number, std::u32string_view suffix = U"") const override;
-    std::u32string to_currency(char32_t currency, std::variant<int32_t, float> number) const override;
-    std::u32string to_fraction(int32_t numerator, int32_t denominator) const override;
-    std::u32string to_day(uint32_t day) const override;
-    std::u32string to_month(uint32_t month) const override;
-    std::u32string to_year(uint32_t year) const override;
-    bool is_ordinal_suffix(std::u32string_view suffix) const override;
+  Num2Word();
+
+protected:
+  using Context = processor::Context;
+  using Match = processor::num2word::Match;
+  using Number = processor::num2word::Number;
+
+  // Resolution
+  bool accept(const Match& match, const Context& context) const override;
+  NumberReading resolve(const Match& match, const Context& context) const override;
+
+  // Readings
+  std::u32string read_number(const Number& number, const NumberReading& reading) const override;
+  std::u32string read_code_digits(std::u32string_view digits) const override;
+  std::u32string read_date(const Match& match, const NumberReading& reading) const override;
+  std::u32string read_phone(const Match& match, const NumberReading& reading) const override;
+  std::u32string read_fraction(const Match& match, const NumberReading& reading) const override;
+  std::u32string read_time(const Match& match, const NumberReading& reading) const override;
+  std::u32string read_ordinal(const Match& match, const NumberReading& reading) const override;
+  std::u32string read_decade(const Match& match, const NumberReading& reading) const override;
+  std::u32string read_money(const Number& number, const Match& match, const NumberReading& reading) const override;
+  std::u32string read_scale(const Number& number, const Match& match, const NumberReading& reading) const override;
+  std::u32string read_percent(const Number& number, const Match& match, const NumberReading& reading) const override;
+  std::u32string read_unit(const Number& number, const Match& match, const NumberReading& reading) const override;
+  std::u32string read_range(const Number& from, const Number& to, const NumberReading& reading) const override;
+  std::optional<std::u32string> plural(std::u32string_view words) const override;
+
+private:
+  // An integer as a year, a day of a month or a count, digit by digit if it is a code
+  std::u32string read_integer(const Number& number, const NumberReading& reading) const;
 };
 
 } // namespace phonemis::en

package/third-party/common/phonemis/src/phonemis/lang/en/pipeline.h

@@ -8,9 +8,8 @@
 #include <phonemis/base/config.h>
 #include <phonemis/base/ipipeline.h>
 #include <phonemis/base/phonemizer/hybrid_phonemizer.h>
+#include <phonemis/base/processor/preprocessor.h>
 #include <phonemis/base/processor/processor.h>
-#include <phonemis/base/processor/sanitizer_layer.h>
-#include <phonemis/base/processor/trim_layer.h>
 #include <phonemis/base/tagger/hmm_tagger.h>
 #include <phonemis/base/tokenizer/tokenizer.h>
 #include <phonemis/utils/conversions.h>
@@ -27,14 +26,8 @@ public:
         phonemizer_(config.phonemizer) {
 
     // 1. Setup Preprocessing layers
-    preprocessor_.add_layer(std::make_unique<processor::TrimLayer>());
-    preprocessor_.add_layer(std::make_unique<Num2Word>());
-    preprocessor_.add_layer(std::make_unique<processor::SanitizerLayer>(
-        [](char32_t) { return true; },
-        [](char32_t c) {
-          auto it = constants::kSanitizerReplacements.find(c);
-          return it != constants::kSanitizerReplacements.end() ? it->second : c;
-        }));
+    preprocessor_ = processor::make_preprocessor<Num2Word>(
+        config.preprocessor, {.terms = &constants::term2word::kTerms, .symbols = &constants::sym2word::kSymbols});
 
     // 2. Setup PoS tagger if specified
     if (config.tagger.has_value()) {