react-native-executorch 0.10.5 → 0.10.7

package/third-party/common/phonemis/src/phonemis/base/tagger/hmm_tagger.h

@@ -6,7 +6,7 @@
 #include <span>
 #include <string>
 #include <unordered_map>
-#include <unordered_set>
+#include <vector>
 
 namespace phonemis::tagger {
 
@@ -34,15 +34,22 @@ protected:
    */
   virtual bool should_lowercase(std::u32string_view word) const { return false; }
 
-  // Set of all possible tags extracted from the model
-  std::unordered_set<Tag> tags_;
+  // Tags are stored as a dense, sorted integer index: tag_names_[id] is the
+  // tag string. Every probability table below is indexed by this id, so the
+  // Viterbi hot path uses array lookups instead of hashing tag strings.
+  std::vector<Tag> tag_names_;
 
-  // Probability tables
-  std::unordered_map<Tag, double> start_probs_;
-  // Tag -> Word -> Probability (UTF-8 word keys to save memory)
-  std::unordered_map<Tag, std::unordered_map<std::string, double>> emission_probs_;
-  // PrevTag -> NextTag -> Probability
-  std::unordered_map<Tag, std::unordered_map<Tag, double>> transition_probs_;
+  // start_probs_[id] : log P(first tag == id)
+  std::vector<double> start_probs_;
+
+  // transition_probs_[prev * T + curr] : log P(curr | prev), with
+  // T == tag_names_.size(). Missing transitions are pre-filled with a small
+  // log-epsilon, so the recursion needs no per-cell presence check.
+  std::vector<double> transition_probs_;
+
+  // emission_probs_[id] : word -> log P(word | tag == id). Words stay
+  // string-keyed because the observation vocabulary is open-ended.
+  std::vector<std::unordered_map<std::string, double>> emission_probs_;
 };
 
 } // namespace phonemis::tagger

package/third-party/common/phonemis/src/phonemis/base/tokenizer/tokenizer.cpp

@@ -14,13 +14,13 @@ Tokenizer::Tokenizer(const split::Rules* split_rules,
 std::vector<Token>
 Tokenizer::tokenize(std::u32string_view input) const {
   // A resulting list of tokens
-	std::vector<Token> tokens;
+  std::vector<Token> tokens;
   tokens.reserve(input.size() / 5);
 
   // A dynamic buffer to collect characters to be processed.
   // Since we try to avoid copying strings, we represent the buffer as a view
   // parametrized by offset (start) and length (len), similarly to string_view.
-	size_t currw_offset = 0, currw_len = 0;
+  size_t currw_offset = 0, currw_len = 0;
 
   for (size_t idx = 0; idx < input.size(); ++idx) {
     char32_t c = input[idx];
@@ -30,13 +30,13 @@ Tokenizer::tokenize(std::u32string_view input) const {
       // and there cannot be 2 or more consecutive white characters.
 
       // If we have a pending word, process it
-			if (currw_len > 0) {  // equivalent to !curr_word.empty()
-				process_phrase(std::u32string_view(input.data() + currw_offset, currw_len), tokens);
-				currw_len = 0;  // resets the current word view
+	  if (currw_len > 0) {  // equivalent to !curr_word.empty()
+		  process_phrase(std::u32string_view(input.data() + currw_offset, currw_len), tokens);
+		  currw_len = 0;  // resets the current word view
 
-				// The right-most token always gets the trailing white space flag, except for the last token in the sequence
-				tokens.back().whitespace = true;
-			}
+		  // The right-most token always gets the trailing white space flag, except for the last token in the sequence
+		  tokens.back().whitespace = true;
+	  }
     } else {
       currw_offset = currw_len == 0 ? idx : currw_offset;
       currw_len++;
@@ -97,13 +97,13 @@ void Tokenizer::process_phrase(std::u32string_view word,
 
 void Tokenizer::process_chunk(std::u32string_view chunk, 
                               std::vector<Token>& token_vec) const {
-  // Edge case - an empty chunk/word ("")
+    // Edge case - an empty chunk/word ("")
 	if (chunk.empty()) return;
 
 	// Special word set lookup
 	// If an entire chunk is a special word/phrase, we should return it without
 	// further divisions.
-  // Note that the lookup is not case sensitive.
+    // Note that the lookup is not case sensitive.
 	if (exceptions_ && is_exception(strings::to_lower(chunk))) {
 		token_vec.push_back({std::u32string(chunk)});
 		return;
@@ -175,8 +175,9 @@ void Tokenizer::process_chunk(std::u32string_view chunk,
 }
 
 split::Rule Tokenizer::get_rule(char32_t c) const {
-  return rules_ && rules_->contains(c) ?
-         rules_->at(c) : split::Rule::TOTAL_DIVIDE;
+  // A character has a rule exactly when it is a soft separator; everything
+  // else falls back to splitting on both sides.
+  return is_soft_separator(c) ? rules_->at(c) : split::Rule::TOTAL_DIVIDE;
 }
 
 bool Tokenizer::is_soft_separator(char32_t c) const {

package/third-party/common/phonemis/src/phonemis/lang/de/constants.h

@@ -1,48 +1,212 @@
 #pragma once
 
+#include <phonemis/base/processor/layers/num2word/types.h>
+#include <phonemis/base/processor/common/conditions.h>
+#include <phonemis/base/processor/layers/num2word/notation.h>
+#include <phonemis/base/processor/layers/term2word/types.h>
 #include <phonemis/base/tokenizer/types.h>
 
+#include <array>
 #include <cstdint>
 #include <map>
 #include <string>
+#include <string_view>
 #include <unordered_map>
 #include <unordered_set>
+#include <utility>
 #include <vector>
 
 namespace phonemis::de::constants {
 
 // --- Number to Word Normalization ---
 namespace num2word {
-  inline const std::unordered_map<int32_t, std::u32string> kCardinals = {
-    {0,  U"null"},     {1,  U"eins"},      {2,  U"zwei"},      {3,  U"drei"},      {4,  U"vier"},
-    {5,  U"fünf"},     {6,  U"sechs"},     {7,  U"sieben"},    {8,  U"acht"},      {9,  U"neun"},
-    {10, U"zehn"},     {11, U"elf"},       {12, U"zwölf"},     {13, U"dreizehn"},  {14, U"vierzehn"},
-    {15, U"fünfzehn"}, {16, U"sechzehn"},  {17, U"siebzehn"},  {18, U"achtzehn"},  {19, U"neunzehn"},
-    {20, U"zwanzig"},  {30, U"dreißig"},   {40, U"vierzig"},   {50, U"fünfzig"},   {60, U"sechzig"},
-    {70, U"siebzig"},  {80, U"achtzig"},   {90, U"neunzig"}
-  };
+  using ::phonemis::processor::num2word::DateFormat;
+  using ::phonemis::processor::num2word::Forms;
+  using ::phonemis::processor::num2word::Gender;
+  using ::phonemis::processor::num2word::Joiners;
+  using ::phonemis::processor::num2word::Keys;
+  using ::phonemis::processor::num2word::keys_of;
+  using ::phonemis::processor::num2word::Notation;
+  using ::phonemis::processor::num2word::SuffixFormat;
 
-  inline const std::map<std::int64_t, std::u32string> kLargeCardinals = {
-    {100LL,           U"hundert"},
-    {1000LL,          U"tausend"},
-    {1000000LL,       U"Million"},
-    {1000000000LL,    U"Milliarde"}
+  inline constexpr std::array<std::u32string_view, 20> kOnes = {
+    U"null",   U"eins",    U"zwei",    U"drei",      U"vier",      U"fünf",      U"sechs",    U"sieben",    U"acht",     U"neun",
+    U"zehn",   U"elf",     U"zwölf",   U"dreizehn",  U"vierzehn",  U"fünfzehn",  U"sechzehn", U"siebzehn",  U"achtzehn", U"neunzehn"
   };
 
-  inline const std::unordered_map<char32_t, std::u32string> kCurrencies = {
-    {U'$', U"Dollar"}, {U'€', U"Euro"}, {U'£', U"Pfund"}
+  inline constexpr std::array<std::u32string_view, 10> kTens = {
+    U"", U"", U"zwanzig", U"dreißig", U"vierzig", U"fünfzig", U"sechzig", U"siebzig", U"achtzig", U"neunzig"
   };
 
   inline const std::vector<std::u32string> kMonths = {
-    U"",         U"Januar",   U"Februar",   U"März",      U"April",    U"Mai",      U"Juni",
-    U"Juli",     U"August",   U"September", U"Oktober",   U"November", U"Dezember"
+    U"Januar", U"Februar", U"März",      U"April",   U"Mai",      U"Juni",
+    U"Juli",   U"August",  U"September", U"Oktober", U"November", U"Dezember"
+  };
+
+  // Ordinal stems which differ from their cardinals: erst, dritt, siebt, acht
+  inline const std::unordered_map<std::u32string, std::u32string> kOrdinalStems = {
+    {U"eins", U"ers"}, {U"drei", U"drit"}, {U"sieben", U"sieb"}, {U"acht", U"ach"}
+  };
+
+  inline const std::unordered_map<std::u32string, std::u32string> kCurrencies = {
+    {U"€", U"Euro"}, {U"Euro", U"Euro"}, {U"$", U"Dollar"}, {U"Dollar", U"Dollar"}, {U"£", U"Pfund"},
+    {U"CHF", U"Franken"}
+  };
+
+  inline const std::unordered_map<std::u32string, Forms> kScales = {
+    {U"Mio.", {U"Million", U"Millionen"}},       {U"Million", {U"Million", U"Millionen"}},
+    {U"Millionen", {U"Million", U"Millionen"}},  {U"Mrd.", {U"Milliarde", U"Milliarden"}},
+    {U"Milliarde", {U"Milliarde", U"Milliarden"}}, {U"Milliarden", {U"Milliarde", U"Milliarden"}},
+    {U"Tsd.", {U"tausend", U"tausend"}},         {U"k", {U"tausend", U"tausend"}}
+  };
+
+  inline const std::unordered_map<std::u32string, Forms> kUnits = {
+    {U"km", {U"Kilometer", U"Kilometer"}},       {U"m", {U"Meter", U"Meter"}},
+    {U"cm", {U"Zentimeter", U"Zentimeter"}},     {U"mm", {U"Millimeter", U"Millimeter"}},
+    {U"nm", {U"Nanometer", U"Nanometer"}},       {U"m²", {U"Quadratmeter", U"Quadratmeter"}},
+    {U"km²", {U"Quadratkilometer", U"Quadratkilometer"}}, {U"m³", {U"Kubikmeter", U"Kubikmeter"}},
+    {U"ha", {U"Hektar", U"Hektar"}},             {U"kg", {U"Kilogramm", U"Kilogramm"}},
+    {U"g", {U"Gramm", U"Gramm"}},                {U"mg", {U"Milligramm", U"Milligramm"}},
+    {U"t", {U"Tonne", U"Tonnen"}},               {U"l", {U"Liter", U"Liter"}},
+    {U"ml", {U"Milliliter", U"Milliliter"}},     {U"TB", {U"Terabyte", U"Terabyte"}},
+    {U"GB", {U"Gigabyte", U"Gigabyte"}},         {U"MB", {U"Megabyte", U"Megabyte"}},
+    {U"Hz", {U"Hertz", U"Hertz"}},               {U"kHz", {U"Kilohertz", U"Kilohertz"}},
+    {U"MHz", {U"Megahertz", U"Megahertz"}},      {U"GHz", {U"Gigahertz", U"Gigahertz"}},
+    {U"W", {U"Watt", U"Watt"}},                  {U"kW", {U"Kilowatt", U"Kilowatt"}},
+    {U"kWh", {U"Kilowattstunde", U"Kilowattstunden"}}, {U"V", {U"Volt", U"Volt"}},
+    {U"km/h", {U"Kilometer pro Stunde", U"Kilometer pro Stunde"}}, {U"°C", {U"Grad Celsius", U"Grad Celsius"}},
+    {U"°", {U"Grad", U"Grad"}},                  {U"h", {U"Stunde", U"Stunden"}},
+    {U"Std.", {U"Stunde", U"Stunden"}},          {U"min", {U"Minute", U"Minuten"}},
+    {U"Min.", {U"Minute", U"Minuten"}},          {U"s", {U"Sekunde", U"Sekunden"}},
+    {U"Sek.", {U"Sekunde", U"Sekunden"}},        {U"ct", {U"Cent", U"Cent"}},
+    {U"PS", {U"PS", U"PS"}}
+  };
+
+  inline const std::map<std::pair<uint64_t, uint64_t>, std::u32string> kFractions = {
+    {{1, 2}, U"ein halb"}, {{1, 3}, U"ein Drittel"}, {{2, 3}, U"zwei Drittel"}, {{1, 4}, U"ein Viertel"},
+    {{3, 4}, U"drei Viertel"}
+  };
+
+  // Determiners after which an ordinal takes a weak ending: am dritten, der dritte (nominative)
+  inline const std::unordered_set<std::u32string> kWeak = {
+    U"den", U"dem", U"des", U"am", U"im", U"vom", U"zum", U"zur", U"beim", U"ins", U"diesem", U"diesen", U"dieses",
+    U"jedem", U"jeden", U"einen", U"einem", U"einer", U"eines", U"keinen", U"keinem", U"keiner", U"keines",
+    U"meinen", U"meinem", U"meiner", U"seinen", U"seinem", U"seiner", U"ihren", U"ihrem", U"ihrer", U"unseren",
+    U"unserem", U"unserer"
+  };
+  inline const std::unordered_set<std::u32string> kWeakNominative = {
+    U"der", U"die", U"das", U"diese", U"dieser", U"jede", U"jeder", U"jedes", U"eine", U"keine", U"meine", U"seine",
+    U"ihre", U"unsere"
+  };
+
+  // ein-words, after which an ordinal takes a mixed ending: ein zweiter, ein drittes
+  inline const std::unordered_set<std::u32string> kMixed = {
+    U"ein", U"kein", U"mein", U"dein", U"sein", U"ihr", U"unser", U"euer"
   };
 
-  inline const std::unordered_set<std::u32string> kOrdinalSuffixes = {
-    U".", U"te", U"ste", U"ten", U"sten", U"ter", U"ster", U"tes", U"stes", U"tem", U"stem"
+  // Prepositions by the case they govern
+  inline const std::unordered_set<std::u32string> kDative = {
+    U"mit", U"nach", U"bei", U"seit", U"von", U"zu", U"aus", U"gegenüber", U"in", U"an", U"auf", U"unter", U"über",
+    U"vor", U"hinter", U"neben", U"zwischen", U"ab"
+  };
+  inline const std::unordered_set<std::u32string> kAccusative = {U"für", U"durch", U"gegen", U"ohne", U"um", U"bis"};
+  inline const std::unordered_set<std::u32string> kGenitive = {
+    U"wegen", U"während", U"trotz", U"statt", U"innerhalb", U"außerhalb"
   };
+
+  // Words before a date (am 12. 3.), and before a year (im Jahr 1945, and months)
+  inline const std::unordered_set<std::u32string> kDateCues = {U"am", U"vom", U"zum", U"bis", U"ab", U"seit", U"den", U"der"};
+  inline const std::unordered_set<std::u32string> kYearCues = {
+    U"jahr", U"jahre", U"jahren", U"im", U"seit", U"bis", U"von", U"ab", U"anfang", U"ende", U"mitte", U"sommer",
+    U"winter", U"frühjahr", U"herbst", U"frühling", U"anno", U"januar", U"februar", U"märz", U"april", U"mai",
+    U"juni", U"juli", U"august", U"september", U"oktober", U"november", U"dezember"
+  };
+
+  // Nouns which stay singular after a number: 1500 Stück. Like plural nouns, they make a 4-digit number a count.
+  inline const std::unordered_set<std::u32string> kMeasureWords = {
+    U"Stück", U"Mal", U"Prozent", U"Grad", U"Euro", U"Dollar", U"Pfund", U"Kilo", U"Gramm", U"Paar", U"Cent",
+    U"Franken"
+  };
+
+  // Noun gender by ending: the exceptions that matter after numbers (matched as compound endings), then the rules
+  inline const std::vector<std::pair<std::u32string, Gender>> kGenders = {
+    {U"jahrhundert", Gender::NEUTER}, {U"jahrtausend", Gender::NEUTER}, {U"semester", Gender::NEUTER},
+    {U"mitglied", Gender::NEUTER},    {U"kapitel", Gender::NEUTER},     {U"quartal", Gender::NEUTER},
+    {U"prozent", Gender::NEUTER},     {U"zimmer", Gender::NEUTER},      {U"rennen", Gender::NEUTER},
+    {U"person", Gender::FEMININE},    {U"gramm", Gender::NEUTER},       {U"stück", Gender::NEUTER},
+    {U"spiel", Gender::NEUTER},       {U"leben", Gender::NEUTER},       {U"liter", Gender::MASCULINE},
+    {U"meter", Gender::MASCULINE},    {U"monat", Gender::MASCULINE},    {U"kunde", Gender::MASCULINE},
+    {U"junge", Gender::MASCULINE},    {U"kilo", Gender::NEUTER},        {U"jahr", Gender::NEUTER},
+    {U"kind", Gender::NEUTER},        {U"glas", Gender::NEUTER},        {U"haus", Gender::NEUTER},
+    {U"land", Gender::NEUTER},        {U"buch", Gender::NEUTER},        {U"bild", Gender::NEUTER},
+    {U"ende", Gender::NEUTER},        {U"pfund", Gender::NEUTER},       {U"euro", Gender::MASCULINE},
+    {U"name", Gender::MASCULINE},     {U"käse", Gender::MASCULINE},     {U"grad", Gender::MASCULINE},
+    {U"mal", Gender::NEUTER},         {U"tor", Gender::NEUTER},         {U"ei", Gender::NEUTER}
+  };
+  inline const std::vector<std::u32string> kFeminineEndings = {
+    U"e", U"ung", U"heit", U"keit", U"schaft", U"ion", U"tät", U"ik", U"ur", U"enz", U"anz", U"ei", U"in", U"ie"
+  };
+  inline const std::vector<std::u32string> kNeuterEndings = {U"chen", U"lein", U"ment", U"um", U"tum", U"ma", U"o"};
+
+  // --- Notation: how numbers are written ---
+  inline const Notation kNotation = {
+    .group_separators = U". \u00a0\u202f", .decimal_separators = U",.", .time_separators = U":.", .time_suffixes = {U"Uhr"},
+    .currencies = keys_of(kCurrencies), .scales = keys_of(kScales), .units = keys_of(kUnits), .fractions = true
+  };
+
+  // Words between numbers: 3.8.10, 10.0.0.51, 10-15, 1920x1080, 2:1, -5, 2 + 2
+  inline const Joiners kJoiners = {
+    .point = U"Komma", .dot = U"Punkt", .range = U" bis ", .by = U"mal", .score = U" zu ", .minus = U"minus", .plus = U"plus"
+  };
+
+  // 1.5.2024, 1.5., 12. 3.: the year is optional, a dot closes the month
+  inline const DateFormat kDateFormat = {
+    .separators = U".", .boundaries = U".,", .year_required = false, .closing_separator = true, .spaced = true
+  };
+
+  // 90er, 90ern, 90ers
+  inline const Keys kDecadeSuffixes = {U"er", U"ern", U"ers"};
+  inline const SuffixFormat kDecadeFormat = {.boundaries = U".,-"};
 } // namespace num2word
 
+// --- Symbols to Word Normalization ---
+namespace sym2word {
+  using ::phonemis::processor::Readings;
+  using ::phonemis::processor::after_any;
+
+  // Words before "§" which take its plural: im § 5 (in dem Paragrafen)
+  inline const std::unordered_set<std::u32string> kParagraphPlurals = {
+    U"im", U"vom", U"zum", U"beim", U"dem", U"des", U"den"
+  };
+
+  inline const Readings kSymbols = {
+    {U"&", {U"und"}},       {U"€", {U"Euro"}},      {U"$", {U"Dollar"}},    {U"£", {U"Pfund"}},    {U"+", {U"plus"}},
+    {U"=", {U"gleich"}},    {U"@", {U"at"}},        {U"%", {U"Prozent"}},   {U"°", {U"Grad"}},     {U"#", {U"Hashtag"}},
+    {U"²", {U"Quadrat"}},   {U"³", {U"hoch drei"}}, {U"¹", {U"hoch eins"}}, {U"π", {U"Pi"}},       {U"×", {U"mal"}},
+    {U"½", {U"ein halb"}},  {U"§§", {U"Paragrafen"}}, {U"§", {{U"Paragrafen", after_any(kParagraphPlurals)}, U"Paragraf"}}
+  };
+} // namespace sym2word
+
+// --- Terms to Word Normalization ---
+namespace term2word {
+  using ::phonemis::processor::before_number;
+  using ::phonemis::processor::term2word::Terms;
+
+  inline const Terms kTerms = {
+    .abbreviations = {
+      // Before a number only: Abs. 5
+      {U"Abs.", {{U"Absatz", before_number}}},
+      {U"Nr.", {{U"Nummer", before_number}}},
+      {U"S.", {{U"Seite", before_number}}},
+      {U"Art.", {{U"Artikel", before_number}}},
+      {U"Kap.", {{U"Kapitel", before_number}}},
+      {U"ca.", {{U"circa", before_number}}},
+      {U"Bd.", {{U"Band", before_number}}},
+    },
+    .web = {.dot = U"Punkt", .at = U"at", .slash = U"Schrägstrich"}
+  };
+} // namespace term2word
+
 // --- Tokenization Rules ---
 namespace tokenizer {
   using ::phonemis::tokenizer::split::Rule;