react-native-executorch 0.10.6 → 0.10.7

package/third-party/common/phonemis/src/phonemis/lang/de/num2word.h

@@ -1,29 +1,54 @@
 #pragma once
 
-#include <phonemis/base/processor/num2word/layer.h>
+#include <phonemis/base/processor/layers/num2word/layer.h>
+
 #include <string>
-#include <variant>
 
 namespace phonemis::de {
 
-class Num2Word : public processor::num2word::Num2WordLayer {
+using processor::num2word::Case;
+using processor::num2word::Gender;
+
+// How the ending of an ordinal follows the word before it: am dritten (weak), ein zweiter (mixed), dritter (strong)
+enum class Declension { WEAK = 0, MIXED, STRONG };
+
+// How a German number is read, from the words around it.
+struct NumberReading {
+  Case grammatical_case = Case::NOMINATIVE; // Given by the word before: mit einer Stimme, am dritten
+  Gender gender = Gender::MASCULINE;        // Of the noun after: eine Stunde, der dritte Platz
+  Declension declension = Declension::STRONG;
+  bool article = false;    // "1" before a noun is said like an article: eine Stunde
+  bool year = true;        // A 4-digit number reads as a year (1998 Abi), unless it counts a noun (1945 Einwohner)
+  bool capitalized = true; // A decade is a noun (die Neunziger), but an adjective before Jahre (neunziger Jahre)
+};
+
+class Num2Word : public processor::num2word::Num2WordLayer<NumberReading> {
 public:
-    using Num2WordLayer::Num2WordLayer;
-
-    std::u32string transform(std::u32string_view input) const override;
-    char32_t decimal_separator() const override { return U','; }
-    std::u32string to_cardinal_int(int32_t number) const override;
-    std::u32string to_cardinal_float(float number, std::u32string_view repr) const override;
-    std::u32string to_ordinal_int(int32_t number, std::u32string_view suffix = U"") const override;
-    std::u32string to_currency(char32_t currency, std::variant<int32_t, float> number) const override;
-    std::u32string to_fraction(int32_t numerator, int32_t denominator) const override;
-    std::u32string to_day(uint32_t day) const override;
-    std::u32string to_month(uint32_t month) const override;
-    std::u32string to_year(uint32_t year) const override;
-    bool is_ordinal_suffix(std::u32string_view suffix) const override;
+  Num2Word();
 
 protected:
-    std::u32string verbalize(const processor::num2word::StringifiedNumber& number) const override;
+  using Context = processor::Context;
+  using Match = processor::num2word::Match;
+  using Number = processor::num2word::Number;
+
+  // Resolution
+  bool accept(const Match& match, const Context& context) const override;
+  NumberReading resolve(const Match& match, const Context& context) const override;
+
+  // Readings
+  std::u32string read_number(const Number& number, const NumberReading& reading) const override;
+  std::u32string read_code_digits(std::u32string_view digits) const override;
+  std::u32string read_date(const Match& match, const NumberReading& reading) const override;
+  std::u32string read_phone(const Match& match, const NumberReading& reading) const override;
+  std::u32string read_fraction(const Match& match, const NumberReading& reading) const override;
+  std::u32string read_time(const Match& match, const NumberReading& reading) const override;
+  std::u32string read_ordinal(const Match& match, const NumberReading& reading) const override;
+  std::u32string read_decade(const Match& match, const NumberReading& reading) const override;
+  std::u32string read_compound(const Match& match, const NumberReading& reading) const override;
+  std::u32string read_money(const Number& number, const Match& match, const NumberReading& reading) const override;
+  std::u32string read_scale(const Number& number, const Match& match, const NumberReading& reading) const override;
+  std::u32string read_percent(const Number& number, const Match& match, const NumberReading& reading) const override;
+  std::u32string read_unit(const Number& number, const Match& match, const NumberReading& reading) const override;
 };
 
 } // namespace phonemis::de

package/third-party/common/phonemis/src/phonemis/lang/de/patterns.cpp

@@ -0,0 +1,56 @@
+#include "patterns.h"
+#include <phonemis/base/processor/layers/num2word/primitives.h>
+#include <phonemis/utils/unicode.h>
+
+#include <vector>
+
+namespace phonemis::de {
+
+using namespace utils;
+using namespace processor;
+using namespace processor::num2word;
+
+std::optional<Match> PhonePattern::match(std::u32string_view text, size_t position) const {
+  if (!is_clear_before(text, position, U".,") || text[position] != U'0') {
+    return std::nullopt;
+  }
+
+  // The area code and the number: written together (0301234567), or apart (030 1234567, 030/1234567)
+  size_t end = digits_end(text, position);
+  size_t length = end - position;
+  if (length < 6) {
+    bool apart = length >= 3 && end < text.size() && (text[end] == U' ' || text[end] == U'/' || text[end] == U'-') &&
+                 digits_end(text, end + 1) >= end + 4;
+    if (!apart) {
+      return std::nullopt;
+    }
+    end = digits_end(text, end + 1);
+  }
+
+  // Then any further groups: 0800-123 45 67. Each could end the number, if nothing continues it there.
+  std::vector<size_t> ends = {end};
+  while (ends.back() + 1 < text.size() && (text[ends.back()] == U' ' || text[ends.back()] == U'-') &&
+         unicode::isdigit(text[ends.back() + 1])) {
+    ends.push_back(digits_end(text, ends.back() + 1));
+  }
+  for (auto it = ends.rbegin(); it != ends.rend(); ++it) {
+    // Not followed by a digit, even after a word character, a dot or a comma
+    bool continued = is_digit_after(text, *it) || (!is_clear_after(text, *it, U".,") && is_digit_after(text, *it + 1));
+    if (!continued) {
+      Match phone{.kind = Kind::PHONE, .span = {position, *it}};
+      for (size_t i = position; i < *it;) {
+        size_t digits = digits_end(text, i);
+        if (digits > i) {
+          phone.numbers.push_back(digits_at(text, i));
+          i = digits;
+        } else {
+          i++;
+        }
+      }
+      return phone;
+    }
+  }
+  return std::nullopt;
+}
+
+} // namespace phonemis::de

package/third-party/common/phonemis/src/phonemis/lang/de/patterns.h

@@ -0,0 +1,15 @@
+#pragma once
+
+#include <phonemis/base/processor/layers/num2word/pattern.h>
+
+namespace phonemis::de {
+
+// 030 1234567, 0800-123 45 67: an area code starting with 0, then the number, optionally in groups.
+class PhonePattern : public processor::num2word::Pattern {
+public:
+  PhonePattern() : Pattern(processor::num2word::Kind::PHONE) {}
+
+  std::optional<processor::num2word::Match> match(std::u32string_view text, size_t position) const override;
+};
+
+} // namespace phonemis::de

package/third-party/common/phonemis/src/phonemis/lang/de/pipeline.h

@@ -7,8 +7,8 @@
 #include <phonemis/base/ipipeline.h>
 #include <phonemis/base/phonemizer/hybrid_phonemizer.h>
 #include <phonemis/base/phonemizer/lexicon_phonemizer.h>
+#include <phonemis/base/processor/preprocessor.h>
 #include <phonemis/base/processor/processor.h>
-#include <phonemis/base/processor/trim_layer.h>
 #include <phonemis/base/tokenizer/tokenizer.h>
 #include <phonemis/utils/conversions.h>
 
@@ -24,8 +24,8 @@ public:
         phonemizer_(config.phonemizer) {
 
     // 1. Setup Preprocessing layers
-    preprocessor_.add_layer(std::make_unique<processor::TrimLayer>());
-    preprocessor_.add_layer(std::make_unique<Num2Word>());
+    preprocessor_ = processor::make_preprocessor<Num2Word>(
+        config.preprocessor, {.terms = &constants::term2word::kTerms, .symbols = &constants::sym2word::kSymbols});
   }
 
   // Performs a preprocessing stage of the pipeline.

package/third-party/common/phonemis/src/phonemis/lang/en/constants.h

@@ -1,55 +1,210 @@
 #pragma once
 
+#include <phonemis/base/processor/layers/num2word/types.h>
+#include <phonemis/base/processor/common/conditions.h>
+#include <phonemis/base/processor/layers/num2word/notation.h>
+#include <phonemis/base/processor/layers/term2word/types.h>
 #include <phonemis/base/tokenizer/types.h>
 
+#include <array>
 #include <cstdint>
 #include <map>
 #include <string>
 #include <unordered_map>
 #include <unordered_set>
+#include <utility>
 #include <vector>
 
 namespace phonemis::en::constants {
 
 // --- Number to Word Normalization ---
 namespace num2word {
-  // Cards map: basic number -> word
-  inline const std::unordered_map<int32_t, std::u32string> kCardinals = {
-    {0,  U"zero"},     {1,  U"one"},      {2,  U"two"},       {3,  U"three"},    {4,  U"four"},
-    {5,  U"five"},     {6,  U"six"},      {7,  U"seven"},     {8,  U"eight"},    {9,  U"nine"},
-    {10, U"ten"},      {11, U"eleven"},   {12, U"twelve"},    {13, U"thirteen"}, {14, U"fourteen"},
-    {15, U"fifteen"},  {16, U"sixteen"},  {17, U"seventeen"}, {18, U"eighteen"}, {19, U"nineteen"},
-    {20, U"twenty"},   {30, U"thirty"},   {40, U"forty"},     {50, U"fifty"},    {60, U"sixty"},
-    {70, U"seventy"},  {80, U"eighty"},   {90, U"ninety"}
-  };
-
-  // Ordinal exceptions: cardinal word -> ordinal word
-  inline const std::unordered_map<std::u32string, std::u32string> kOrdinals = {
+  using ::phonemis::processor::num2word::DateFormat;
+  using ::phonemis::processor::num2word::Forms;
+  using ::phonemis::processor::num2word::Joiners;
+  using ::phonemis::processor::num2word::Keys;
+  using ::phonemis::processor::num2word::keys_of;
+  using ::phonemis::processor::num2word::Notation;
+  using ::phonemis::processor::num2word::PhoneFormat;
+  using ::phonemis::processor::num2word::SuffixFormat;
+
+  inline constexpr std::array<std::u32string_view, 20> kOnes = {
+    U"zero",    U"one",     U"two",       U"three",    U"four",     U"five",    U"six",       U"seven",    U"eight",    U"nine",
+    U"ten",     U"eleven",  U"twelve",    U"thirteen", U"fourteen", U"fifteen", U"sixteen",   U"seventeen",U"eighteen", U"nineteen"
+  };
+
+  inline constexpr std::array<std::u32string_view, 10> kTens = {
+    U"", U"", U"twenty", U"thirty", U"forty", U"fifty", U"sixty", U"seventy", U"eighty", U"ninety"
+  };
+
+  // Scale words, largest first
+  inline const std::vector<std::pair<uint64_t, std::u32string>> kScaleWords = {
+    {1000000000000ULL, U"trillion"}, {1000000000ULL, U"billion"}, {1000000ULL, U"million"}, {1000ULL, U"thousand"}
+  };
+
+  // Ordinals which are not formed with "th": cardinal word -> ordinal word
+  inline const std::unordered_map<std::u32string, std::u32string> kIrregularOrdinals = {
     {U"one",   U"first"},  {U"two",   U"second"}, {U"three", U"third"},  {U"five",  U"fifth"},
     {U"eight", U"eighth"}, {U"nine",  U"ninth"},  {U"twelve", U"twelfth"}
   };
 
-  // Large scale names: scale value -> name
-  inline const std::map<std::int64_t, std::u32string> kLargeCardinals = {
-    {100LL,           U"hundred"},
-    {1000LL,          U"thousand"},
-    {1000000LL,       U"million"},
-    {1000000000LL,    U"billion"},
-    {1000000000000LL, U"trillion"}
+  inline const std::vector<std::u32string> kMonthNames = {
+    U"January", U"February", U"March",     U"April",   U"May",      U"June",
+    U"July",    U"August",   U"September", U"October", U"November", U"December"
+  };
+
+  inline const std::unordered_map<std::u32string, Forms> kCurrencies = {
+    {U"$", {U"dollar", U"dollars"}}, {U"€", {U"euro", U"euros"}}, {U"£", {U"pound", U"pounds"}}, {U"¥", {U"yen", U"yen"}}
+  };
+
+  inline const std::unordered_map<std::u32string, std::u32string> kScales = {
+    {U"thousand", U"thousand"}, {U"million", U"million"}, {U"billion", U"billion"}, {U"trillion", U"trillion"},
+    {U"k", U"thousand"},        {U"K", U"thousand"},      {U"M", U"million"},       {U"bn", U"billion"}
+  };
+
+  inline const std::unordered_map<std::u32string, Forms> kUnits = {
+    {U"km",   {U"kilometer", U"kilometers"}},           {U"m",    {U"meter", U"meters"}},
+    {U"cm",   {U"centimeter", U"centimeters"}},         {U"mm",   {U"millimeter", U"millimeters"}},
+    {U"nm",   {U"nanometer", U"nanometers"}},           {U"m²",   {U"square meter", U"square meters"}},
+    {U"km²",  {U"square kilometer", U"square kilometers"}}, {U"m³", {U"cubic meter", U"cubic meters"}},
+    {U"kg",   {U"kilogram", U"kilograms"}},             {U"g",    {U"gram", U"grams"}},
+    {U"mg",   {U"milligram", U"milligrams"}},           {U"lb",   {U"pound", U"pounds"}},
+    {U"lbs",  {U"pound", U"pounds"}},                   {U"oz",   {U"ounce", U"ounces"}},
+    {U"ft",   {U"foot", U"feet"}},                      {U"l",    {U"liter", U"liters"}},
+    {U"L",    {U"liter", U"liters"}},                   {U"ml",   {U"milliliter", U"milliliters"}},
+    {U"mL",   {U"milliliter", U"milliliters"}},         {U"TB",   {U"terabyte", U"terabytes"}},
+    {U"GB",   {U"gigabyte", U"gigabytes"}},             {U"MB",   {U"megabyte", U"megabytes"}},
+    {U"KB",   {U"kilobyte", U"kilobytes"}},             {U"kB",   {U"kilobyte", U"kilobytes"}},
+    {U"Hz",   {U"hertz", U"hertz"}},                    {U"kHz",  {U"kilohertz", U"kilohertz"}},
+    {U"MHz",  {U"megahertz", U"megahertz"}},            {U"GHz",  {U"gigahertz", U"gigahertz"}},
+    {U"W",    {U"watt", U"watts"}},                     {U"kW",   {U"kilowatt", U"kilowatts"}},
+    {U"kWh",  {U"kilowatt-hour", U"kilowatt-hours"}},   {U"V",    {U"volt", U"volts"}},
+    {U"mph",  {U"mile per hour", U"miles per hour"}},   {U"km/h", {U"kilometer per hour", U"kilometers per hour"}},
+    {U"°C",   {U"degree Celsius", U"degrees Celsius"}}, {U"°F",   {U"degree Fahrenheit", U"degrees Fahrenheit"}},
+    {U"°",    {U"degree", U"degrees"}}
+  };
+
+  inline const std::map<std::pair<uint64_t, uint64_t>, std::u32string> kFractions = {
+    {{1, 2}, U"one half"}, {{1, 3}, U"one third"}, {{2, 3}, U"two thirds"}, {{1, 4}, U"one quarter"},
+    {{3, 4}, U"three quarters"}
+  };
+
+  inline const Keys kTimeSuffixes = {U"a.m.", U"p.m.", U"am.", U"pm.", U"am", U"pm", U"AM", U"PM"};
+
+  // Months, also abbreviated (lowercase)
+  inline const std::unordered_set<std::u32string> kMonths = {
+    U"january", U"february", U"march", U"april", U"may", U"june", U"july", U"august", U"september", U"october",
+    U"november", U"december", U"jan", U"feb", U"mar", U"apr", U"jun", U"jul", U"aug", U"sep", U"sept", U"oct",
+    U"nov", U"dec"
+  };
+
+  // Words before a 4-digit number which make it a year: in 1999, since 1999 (and months)
+  inline const std::unordered_set<std::u32string> kYearCues = {
+    U"in", U"of", U"since", U"from", U"by", U"until", U"till", U"circa", U"year", U"the", U"during", U"before",
+    U"after", U"early", U"late", U"mid"
   };
 
-  // Currencies
-  inline const std::unordered_map<char32_t, std::u32string> kCurrencies = {
-    {U'$', U"dollar"}, {U'€', U"euro"}, {U'£', U"pound"}
+  inline const std::unordered_set<std::u32string> kIrregularPlurals = {
+    U"people", U"children", U"men", U"women", U"feet", U"persons"
   };
 
-  // Months
-  inline const std::vector<std::u32string> kMonths = {
-    U"",         U"January",   U"February",  U"March",     U"April",    U"May",      U"June",
-    U"July",     U"August",    U"September", U"October",   U"November", U"December"
+  // Words before a time with a dot, which make it a time: at 10.30
+  inline const std::unordered_set<std::u32string> kTimeWords = {
+    U"at", U"from", U"until", U"till", U"to", U"by", U"around", U"about", U"before", U"after", U"since", U"between",
+    U"and", U"is", U"it's", U"or"
   };
+
+  // Words after an amount that do not make it an adjective ("$5 a month" vs "a $5 bill")
+  inline const std::unordered_set<std::u32string> kStopWords = {
+    U"a", U"an", U"the", U"and", U"or", U"but", U"nor", U"in", U"on", U"at", U"to", U"for", U"of", U"per", U"with",
+    U"from", U"by", U"into", U"than", U"more", U"less", U"each", U"every", U"is", U"are", U"was", U"were", U"be",
+    U"been", U"will", U"would", U"can", U"could", U"should", U"may", U"might", U"must", U"plus", U"minus", U"times",
+    U"anyway", U"now", U"then", U"so", U"that", U"this", U"these", U"those", U"it", U"if", U"when", U"which", U"who",
+    U"what", U"where", U"why", U"how", U"you", U"i", U"we", U"they", U"he", U"she", U"as", U"up", U"down", U"off",
+    U"out", U"over", U"under", U"about", U"around", U"after", U"before", U"back", U"away", U"ago", U"versus", U"vs",
+    U"too", U"also", U"only", U"just", U"even", U"still", U"yet", U"already", U"not", U"no", U"there", U"here",
+    U"all", U"both", U"while", U"because", U"since", U"until", U"though", U"although", U"unless", U"has", U"have",
+    U"had", U"do", U"does", U"did", U"get", U"got", U"instead", U"worth", U"today", U"tomorrow", U"tonight",
+    U"outside", U"inside", U"apart", U"ahead", U"behind", U"above", U"below", U"long", U"wide", U"high", U"tall",
+    U"deep", U"old", U"later", U"earlier", U"overall"
+  };
+
+  // --- Notation: how numbers are written ---
+  inline const Notation kNotation = {
+    .group_separators = U",", .decimal_separators = U".", .time_separators = U":.", .time_suffixes = kTimeSuffixes,
+    .currencies = keys_of(kCurrencies), .scales = keys_of(kScales), .units = keys_of(kUnits), .fractions = true
+  };
+
+  // Words between numbers: 3.8.10, 10.0.0.51, 10-15, 1920x1080, 2:1, -5, 2 + 2
+  inline const Joiners kJoiners = {
+    .point = U"point", .dot = U"dot", .range = U" to ", .by = U"by", .score = U" to ", .minus = U"minus", .plus = U"plus"
+  };
+
+  // 27.03.2026, 27-03-2026: the year has four digits (3.8.10 is a version)
+  inline const DateFormat kDateFormat = {.separators = U".-", .boundaries = U".,/-", .short_year = false};
+
+  // 1-555-123-4567, (555) 123-4567, 555.123.4567, 555 123 4567, 555-1234: the longest first, each its own pass
+  inline const std::vector<PhoneFormat> kPhoneFormats = {
+    {.groups = {{3, 3}, {3, 3}, {4, 4}}, .prefix = U"1-"},
+    {.groups = {{3, 3}, {3, 3}, {4, 4}}, .parenthesized = true},
+    {.groups = {{3, 3}, {3, 3}, {4, 4}}, .separators = U"."},
+    {.groups = {{3, 3}, {3, 3}, {4, 4}}, .separators = U" "},
+    {.groups = {{3, 3}, {4, 4}}}
+  };
+
+  // 21st, 2nd, 3rd, 4th
+  inline const Keys kOrdinalSuffixes = {U"st", U"nd", U"rd", U"th"};
+
+  // 1990s, '90s, 90's
+  inline const Keys kDecadeSuffixes = {U"s"};
+  inline const SuffixFormat kDecadeFormat = {.apostrophes = true, .boundaries = U".,-"};
 } // namespace num2word
 
+// --- Symbols to Word Normalization ---
+namespace sym2word {
+  using ::phonemis::processor::Readings;
+
+  inline const Readings kSymbols = {
+    {U"&", {U"and"}},      {U"+", {U"plus"}},    {U"=", {U"equals"}},  {U"@", {U"at"}},      {U"%", {U"percent"}},
+    {U"°", {U"degrees"}},  {U"#", {U"number"}},  {U"²", {U"squared"}}, {U"³", {U"cubed"}},   {U"π", {U"pi"}},
+    {U"×", {U"times"}},    {U"±", {U"plus or minus"}}, {U"½", {U"one half"}}
+  };
+} // namespace sym2word
+
+// --- Terms to Word Normalization ---
+namespace term2word {
+  using ::phonemis::processor::before_name;
+  using ::phonemis::processor::before_number;
+  using ::phonemis::processor::term2word::Terms;
+
+  inline const Terms kTerms = {
+    .abbreviations = {
+      // Before a number only: No. 5
+      {U"No.", {{U"number", before_number}}},
+      {U"no.", {{U"number", before_number}}},
+      {U"Nos.", {{U"numbers", before_number}}},
+      {U"vol.", {{U"volume", before_number}}},
+      {U"Vol.", {{U"volume", before_number}}},
+      {U"p.", {{U"page", before_number}}},
+      {U"pp.", {{U"pages", before_number}}},
+      {U"ch.", {{U"chapter", before_number}}},
+      {U"Ch.", {{U"chapter", before_number}}},
+      {U"fig.", {{U"figure", before_number}}},
+      {U"Fig.", {{U"figure", before_number}}},
+      // Anywhere
+      {U"e.g.", {U"for example"}},
+      {U"i.e.", {U"that is"}},
+      {U"etc.", {U"et cetera"}},
+      {U"vs.", {U"versus"}},
+      {U"Mr.", {U"Mister"}},
+      {U"Mrs.", {U"Missus"}},
+      {U"Dr.", {{U"Doctor", before_name}}},
+      {U"St.", {{U"Saint", before_name}, U"Street"}},
+    },
+    .web = {.dot = U"dot", .at = U"at", .slash = U"slash"}
+  };
+} // namespace term2word
+
 // --- Tokenization Rules ---
 namespace tokenizer {
   using ::phonemis::tokenizer::split::Rule;
@@ -115,10 +270,6 @@ namespace ipa {
 } // namespace ipa
 
 // --- Symbols & Punctuation Mapping ---
-inline const std::unordered_map<char32_t, char32_t> kSanitizerReplacements = {
-  {U'’', U'\''}
-};
-
 inline const std::unordered_map<char32_t, std::u32string> kAddSymbols = {
   {U'.', U"dot"},
   {U'/', U"slash"}

package/third-party/common/phonemis/src/phonemis/lang/en/hmm_tagger.cpp

@@ -16,13 +16,11 @@ bool HMMTagger::should_lowercase(std::u32string_view word) const {
   Tag best_tag = "XX";
   double max_prob = -std::numeric_limits<double>::infinity();
 
-  for (const auto& [tag, word_map] : emission_probs_) {
-    auto it = word_map.find(utf8_lower_word);
-    if (it != word_map.end()) {
-      if (it->second > max_prob) {
-        max_prob = it->second;
-        best_tag = tag;
-      }
+  for (size_t id = 0; id < emission_probs_.size(); ++id) {
+    auto it = emission_probs_[id].find(utf8_lower_word);
+    if (it != emission_probs_[id].end() && it->second > max_prob) {
+      max_prob = it->second;
+      best_tag = tag_names_[id];
     }
   }