react-native-executorch 0.10.6 → 0.10.7

package/third-party/common/phonemis/src/phonemis/base/processor/layers/num2word/patterns.h

@@ -0,0 +1,285 @@
+#pragma once
+
+#include "notation.h"
+#include "pattern.h"
+
+#include <memory>
+#include <utility>
+#include <vector>
+
+/**
+ * Patterns of numeric expressions
+ *
+ * Generic patterns are added for every language and read its Notation.
+ * Configurable patterns (dates, phones, suffixes) are added by each language with its own format.
+ */
+namespace phonemis::processor::num2word {
+
+// ----------------------------
+// ----- Generic patterns -----
+// ----------------------------
+
+// The generic patterns of a language.
+std::vector<std::unique_ptr<Pattern>> generic_patterns(const Notation& notation);
+
+// 2024-05-12: an ISO date. Its numbers are ordered as day, month, year.
+class IsoDatePattern : public Pattern {
+public:
+  IsoDatePattern() : Pattern(Kind::DATE) {}
+
+  std::optional<Match> match(std::u32string_view text, size_t position) const override;
+};
+
+// 3.8.10, 10.0.0.51; a chain of thousands groups (1.000.000) is left to numbers.
+class VersionPattern : public Pattern {
+public:
+  explicit VersionPattern(const Notation& notation);
+
+  std::optional<Match> match(std::u32string_view text, size_t position) const override;
+
+private:
+  std::u32string group_;
+};
+
+// The " - " of 5 - 3.
+class DifferencePattern : public Pattern {
+public:
+  DifferencePattern() : Pattern(Kind::DIFFERENCE) {}
+
+  std::optional<Match> match(std::u32string_view text, size_t position) const override;
+};
+
+// The "+" of 2 + 2 or 90+6, with the spaces around it.
+class SumPattern : public Pattern {
+public:
+  SumPattern() : Pattern(Kind::SUM) {}
+
+  std::optional<Match> match(std::u32string_view text, size_t position) const override;
+};
+
+// The "-" of 10-15 and 2020-2022. Its numbers are the digits on both sides.
+class RangePattern : public Pattern {
+public:
+  RangePattern() : Pattern(Kind::RANGE) {}
+
+  std::optional<Match> match(std::u32string_view text, size_t position) const override;
+};
+
+// The "x" of 1920x1080, or of 3 m x 4 m (the units are left to UnitPattern).
+class DimensionPattern : public Pattern {
+public:
+  explicit DimensionPattern(const Notation& notation);
+
+  std::optional<Match> match(std::u32string_view text, size_t position) const override;
+
+private:
+  Keys units_;
+};
+
+// 3/4
+class FractionPattern : public Pattern {
+public:
+  FractionPattern() : Pattern(Kind::FRACTION) {}
+
+  std::optional<Match> match(std::u32string_view text, size_t position) const override;
+};
+
+// The minus of -5.
+class SignPattern : public Pattern {
+public:
+  SignPattern() : Pattern(Kind::SIGN) {}
+
+  std::optional<Match> match(std::u32string_view text, size_t position) const override;
+};
+
+// 18:30, 10.30 pm, 21h30: hours and minutes, with an optional suffix.
+class TimePattern : public Pattern {
+public:
+  explicit TimePattern(const Notation& notation);
+
+  std::optional<Match> match(std::u32string_view text, size_t position) const override;
+
+private:
+  // Whether a time may end at a position: not a longer number, nor a measure (1.30 min)
+  bool ends_at(std::u32string_view text, size_t position) const;
+
+  std::u32string separators_;
+  Keys suffixes_;
+  Keys measures_;
+};
+
+// 7pm, 18 Uhr, 21h: an hour marked by its suffix.
+class HourPattern : public Pattern {
+public:
+  explicit HourPattern(const Notation& notation);
+
+  std::optional<Match> match(std::u32string_view text, size_t position) const override;
+
+private:
+  Keys suffixes_;
+};
+
+// The ":" of 2:1.
+class ScorePattern : public Pattern {
+public:
+  ScorePattern() : Pattern(Kind::SCORE) {}
+
+  std::optional<Match> match(std::u32string_view text, size_t position) const override;
+};
+
+// $5, € 1,5 million: a currency symbol before an amount.
+class PrefixedMoneyPattern : public Pattern {
+public:
+  explicit PrefixedMoneyPattern(const Notation& notation);
+
+  std::optional<Match> match(std::u32string_view text, size_t position) const override;
+
+private:
+  Notation notation_;
+  Keys currencies_; // Only the ones which are not words
+};
+
+// 50$, 12,99 zł, 10-15 €, 3 mln zł: an amount, possibly a range, followed by its currency.
+class MoneyPattern : public Pattern {
+public:
+  explicit MoneyPattern(const Notation& notation);
+
+  std::optional<Match> match(std::u32string_view text, size_t position) const override;
+
+private:
+  Notation notation_;
+};
+
+// 3 mln, 500k, 1.5 million
+class ScalePattern : public Pattern {
+public:
+  explicit ScalePattern(const Notation& notation);
+
+  std::optional<Match> match(std::u32string_view text, size_t position) const override;
+
+private:
+  Notation notation_;
+};
+
+// 20%, 10-15 %, 8%ige
+class PercentPattern : public Pattern {
+public:
+  explicit PercentPattern(const Notation& notation);
+
+  std::optional<Match> match(std::u32string_view text, size_t position) const override;
+
+private:
+  Notation notation_;
+};
+
+// 5 kg, 10-15 km, 230 V; a unit of a single capital letter only after a space.
+class UnitPattern : public Pattern {
+public:
+  explicit UnitPattern(const Notation& notation);
+
+  std::optional<Match> match(std::u32string_view text, size_t position) const override;
+
+private:
+  Notation notation_;
+  Keys letters_; // Units of a single capital letter (W, V)
+};
+
+// 21-year-old, 45-metrowe: a number joined to a word by a hyphen.
+class CompoundPattern : public Pattern {
+public:
+  explicit CompoundPattern(const Notation& notation);
+
+  std::optional<Match> match(std::u32string_view text, size_t position) const override;
+
+private:
+  Notation notation_;
+};
+
+// MP3, A380, COVID-19, H.264: a token mixing letters and digits.
+class CodePattern : public Pattern {
+public:
+  CodePattern() : Pattern(Kind::CODE) {}
+
+  std::optional<Match> match(std::u32string_view text, size_t position) const override;
+};
+
+// 42, 1 000,5, .5: any other number.
+class NumberPattern : public Pattern {
+public:
+  explicit NumberPattern(const Notation& notation);
+
+  std::optional<Match> match(std::u32string_view text, size_t position) const override;
+
+private:
+  Notation notation_;
+};
+
+// ---------------------------------
+// ----- Configurable patterns -----
+// ---------------------------------
+
+// A numeric date. Its numbers are ordered as day, month and year (if written).
+class DatePattern : public Pattern {
+public:
+  explicit DatePattern(DateFormat format);
+
+  std::optional<Match> match(std::u32string_view text, size_t position) const override;
+
+private:
+  // The date without (zero) or with its year, if it ends at a valid position
+  std::optional<Match> complete(std::u32string_view text, Match date, size_t position) const;
+
+  DateFormat format_;
+};
+
+// +48 600 123 456, +1 (555) 123-4567: a phone number with a country code. Its numbers are the code, then the groups.
+class InternationalPhonePattern : public Pattern {
+public:
+  InternationalPhonePattern() : Pattern(Kind::INTERNATIONAL_PHONE) {}
+
+  std::optional<Match> match(std::u32string_view text, size_t position) const override;
+};
+
+// A phone number. Its numbers are its groups of digits.
+class PhonePattern : public Pattern {
+public:
+  explicit PhonePattern(PhoneFormat format);
+
+  std::optional<Match> match(std::u32string_view text, size_t position) const override;
+
+private:
+  // The phone number starting with its first group at a position
+  std::optional<Match> groups_at(std::u32string_view text, size_t start, size_t position) const;
+
+  PhoneFormat format_;
+};
+
+// Digits followed by a suffix from a table.
+class SuffixPattern : public Pattern {
+public:
+  SuffixPattern(Kind kind, Keys suffixes, SuffixFormat format = {});
+
+  std::optional<Match> match(std::u32string_view text, size_t position) const override;
+
+private:
+  Keys suffixes_;
+  SuffixFormat format_;
+};
+
+// 3. Platz, 55. sesję: an ordinal written as digits with a dot, followed by a word.
+class DotOrdinalPattern : public Pattern {
+public:
+  /**
+   * @param before_punctuation also before a comma or at the end of the text: lat 70., Punkt 3.
+   * @param sentence_dot leaves the dot to a sentence it also closes.
+   */
+  explicit DotOrdinalPattern(bool before_punctuation = false, bool sentence_dot = false);
+
+  std::optional<Match> match(std::u32string_view text, size_t position) const override;
+
+private:
+  bool before_punctuation_;
+  bool sentence_dot_;
+};
+
+} // namespace phonemis::processor::num2word

package/third-party/common/phonemis/src/phonemis/base/processor/layers/num2word/primitives.cpp

@@ -0,0 +1,64 @@
+#include "../../common/text.h"
+#include "primitives.h"
+#include <phonemis/utils/strings.h>
+#include <phonemis/utils/unicode.h>
+
+#include <algorithm>
+
+namespace phonemis::processor::num2word {
+
+using namespace utils;
+
+Number digits_at(std::u32string_view text, size_t position) {
+  size_t end = digits_end(text, position);
+  return {.span = {position, end}, .integer = std::u32string(text.substr(position, end - position))};
+}
+
+std::vector<Number> numbers_at(std::u32string_view text, size_t position, std::u32string_view group,
+                               std::u32string_view decimal) {
+  size_t run = digits_end(text, position) - position;
+  if (run == 0) {
+    return {};
+  }
+
+  // Integer parts: 1-3 digits followed by whole groups (1 000 000), then a plain run of digits
+  std::vector<Number> integers;
+  if (run <= 3) {
+    size_t end = position + run;
+    while (end + 3 < text.size() && strings::contains(group, text[end]) && digits_end(text, end + 1) >= end + 4) {
+      end += 4;
+    }
+    // A group may not be followed by a digit, nor by a separator and a digit
+    bool complete = end > position + run && !is_digit_after(text, end) &&
+                    !(end < text.size() && strings::contains(group, text[end]) && is_digit_after(text, end + 1));
+    if (complete) {
+      std::u32string digits;
+      std::ranges::copy_if(text.substr(position, end - position), std::back_inserter(digits), unicode::isdigit);
+      integers.push_back({.span = {position, end}, .integer = digits, .grouped = true});
+    }
+  }
+  integers.push_back(digits_at(text, position));
+
+  // Each with its fraction first, then without it
+  std::vector<Number> numbers;
+  for (const Number& integer : integers) {
+    size_t end = integer.span.end;
+    if (end < text.size() && strings::contains(decimal, text[end]) && is_digit_after(text, end + 1)) {
+      Number number = integer;
+      number.span.end = digits_end(text, end + 1);
+      number.fraction = text.substr(end + 1, number.span.end - end - 1);
+      number.decimal_mark = text[end];
+
+      // A fraction shaped like a group is one: the 1.980 of 1.980,6
+      if (!integer.grouped && integer.integer.size() <= 3 && number.fraction.size() == 3 &&
+          strings::contains(group, number.decimal_mark)) {
+        number = {.span = number.span, .integer = integer.integer + number.fraction, .grouped = true};
+      }
+      numbers.push_back(number);
+    }
+    numbers.push_back(integer);
+  }
+  return numbers;
+}
+
+} // namespace phonemis::processor::num2word

package/third-party/common/phonemis/src/phonemis/base/processor/layers/num2word/primitives.h

@@ -0,0 +1,23 @@
+#pragma once
+
+#include "../../common/text.h"
+#include "types.h"
+
+#include <string_view>
+#include <vector>
+
+// Scanning primitives of written numbers, on top of the shared ones (../text.h).
+namespace phonemis::processor::num2word {
+
+// The digits starting at a position, as a number.
+Number digits_at(std::u32string_view text, size_t position);
+
+/**
+ * A written number at a position: 1,000.5 or 1 000,5 or 15.
+ * @returns its possible readings, in the order a regex would try them: grouped first,
+ *          each with and then without its fraction. Empty if no number starts there.
+ */
+std::vector<Number> numbers_at(std::u32string_view text, size_t position, std::u32string_view group,
+                               std::u32string_view decimal);
+
+} // namespace phonemis::processor::num2word

package/third-party/common/phonemis/src/phonemis/base/processor/layers/num2word/types.h

@@ -0,0 +1,94 @@
+#pragma once
+
+#include "../../common/types.h"
+#include "kind.h"
+
+#include <string>
+#include <string_view>
+#include <utility>
+#include <vector>
+
+namespace phonemis::processor::num2word {
+
+using processor::Keys;
+
+// A number as written: "1 000,5" -> integer "1000", fraction "5".
+struct Number {
+  Span span;                 // Where it is written
+  std::u32string integer;    // Digits without group separators; empty in ".5"
+  std::u32string fraction;   // Digits after the decimal mark; empty if none
+  char32_t decimal_mark = 0; // The decimal mark as written, if any
+  bool grouped = false;      // Written with group separators: 1 000, 1,000
+};
+
+// A numeric expression found in a text: its structure only, with no decision on how to read it.
+struct Match {
+  Kind kind = Kind::NUMBER;
+  Span span;
+  std::u32string_view text;    // As written
+  std::vector<Number> numbers; // In written order: one for most kinds, two for ranges, three for dates
+  std::u32string_view unit;    // A unit or currency key ("km", "zł"), or a law word of a DATE ("Ley")
+  std::u32string_view scale;   // A scale key ("mln", "million")
+  std::u32string_view suffix;  // Marks after the number: "st", "er", "." (ordinals), "ige" (8%ige), "pm", "r."
+  char32_t separator = 0;      // The symbol between numbers: '-', ':', '.', '/', 'h', 'x'
+};
+
+// Words read in place of the symbols between numbers.
+struct Joiners {
+  std::u32string point; // 3.8.10: three point eight point ten
+  std::u32string dot;   // 10.0.0.51 (four parts): ten dot zero dot zero dot fifty-one
+  std::u32string range; // 10-15, with its own spaces: " to ", " bis ", "-"
+  std::u32string by;    // 1920x1080
+  std::u32string score; // 2:1, with its own spaces: " to "
+  std::u32string minus; // -5 and 5 - 3
+  std::u32string plus;  // 2 + 2
+};
+
+// The forms of a noun after a number: one dollar, two dollars.
+struct Forms {
+  std::u32string one;
+  std::u32string many;
+};
+
+// Vocabulary of readings: languages build their own readings out of these
+enum class Form { CARDINAL = 0, ORDINAL, YEAR, DIGITS };
+enum class Case { NOMINATIVE = 0, GENITIVE, DATIVE, ACCUSATIVE, INSTRUMENTAL, LOCATIVE };
+enum class Gender { MASCULINE = 0, FEMININE, NEUTER };
+
+// ---------------------------------------------
+// ----- Formats of configurable patterns -----
+// ---------------------------------------------
+
+// How a numeric date is written in a language: 12.05.2024, 12/05/24, 1. 5.
+struct DateFormat {
+  std::u32string separators = U"/";   // Between its parts: the same one twice in a date
+  std::u32string boundaries = U".,/"; // Besides letters, what may not precede it (or a digit after it)
+  bool year_required = true;
+  bool short_year = true;         // Two-digit years: 12/05/24
+  bool closing_separator = false; // A separator also closes the month: 1.5. and 1.5.2024
+  bool spaced = false;            // An optional space after the day: 12. 5.
+  Keys suffixes;                  // Abbreviations after the year: 2024 r.
+};
+
+// How a phone number is written: its groups of digits and what separates them.
+struct PhoneFormat {
+  std::vector<std::pair<size_t, size_t>> groups; // Length range of each group of digits
+  std::u32string separators = U"-";
+  bool optional_separators = false; // 0612345678 as well as 06 12 34 56 78
+  std::u32string prefix;            // An optional literal start: "1-"
+  bool parenthesized = false;       // The first group in parentheses, then an optional space: (555) 123-4567
+  bool leading_zero = false;
+  std::u32string boundaries = U"-"; // Besides letters, what may not stand right before it
+  std::u32string closing = U"-";    // Besides letters, what may not stand right after it
+};
+
+// How a suffix marks a number: 21st, 1 º, 90's, 2020 r.
+struct SuffixFormat {
+  bool space = false;                // An optional space before the suffix
+  bool word_end = true;              // No word character may follow the suffix
+  bool apostrophes = false;          // '90s, 90's: optional apostrophes before the number and the suffix
+  std::u32string boundaries = U".,"; // Besides letters, what may not stand right before the number
+  bool sentence_dot = false;         // A dot closing the suffix is left to a sentence it also closes
+};
+
+} // namespace phonemis::processor::num2word

package/third-party/common/phonemis/src/phonemis/base/processor/layers/sanitizer/layer.cpp

@@ -0,0 +1,19 @@
+#include "layer.h"
+
+namespace phonemis::processor::sanitizer {
+
+SanitizerLayer::SanitizerLayer(Filter filter, Mapper mapper) : filter_(std::move(filter)), mapper_(std::move(mapper)) {}
+
+void SanitizerLayer::apply(Document& document) const {
+  std::u32string_view text = document.text();
+
+  for (size_t i = 0; i < text.size(); ++i) {
+    if (!filter_(text[i])) {
+      document.replace({i, i + 1}, U"");
+    } else if (char32_t c = mapper_(text[i]); c != text[i]) {
+      document.replace({i, i + 1}, std::u32string(1, c));
+    }
+  }
+}
+
+} // namespace phonemis::processor::sanitizer

package/third-party/common/phonemis/src/phonemis/base/processor/{sanitizer_layer.h → layers/sanitizer/layer.h}

@@ -1,31 +1,31 @@
 #pragma once
 
-#include "layer.h"
+#include "../../layer.h"
 
 #include <functional>
-#include <string>
 
-namespace phonemis::processor {
+namespace phonemis::processor::sanitizer {
 
 /**
  * Sanitizer layer is responsible for filtering out (and/or replacing) unrecognizable
  * or undesirable characters in given language.
+ *
+ * It is controlled by a filter function (boolean check) and a mapping function (replacement).
  * 
- * It is controlled by a filter function and a mapping function.
+ * IMPORTANT: Should only be applied after Num2Word and Sym2Word layers.
  */
 class SanitizerLayer : public Layer {
 public:
   using Filter = std::function<bool(char32_t)>;
   using Mapper = std::function<char32_t(char32_t)>;
 
-  SanitizerLayer(Filter filter = [](char32_t c) { return true; },
-                 Mapper mapper = [](char32_t c) { return c; });
+  SanitizerLayer(Filter filter = [](char32_t c) { return true; }, Mapper mapper = [](char32_t c) { return c; });
 
-  std::u32string transform(std::u32string_view input) const override;
+  void apply(Document& document) const override;
 
 private:
   Filter filter_;
   Mapper mapper_;
 };
 
-} // namespace phonemis::processor
+} // namespace phonemis::processor::sanitizer

package/third-party/common/phonemis/src/phonemis/base/processor/layers/sym2word/layer.cpp

@@ -0,0 +1,65 @@
+#include "../../common/constants.h"
+#include "../../common/text.h"
+#include "layer.h"
+#include <phonemis/utils/strings.h>
+#include <phonemis/utils/unicode.h>
+
+namespace phonemis::processor::sym2word {
+
+using namespace utils;
+
+Sym2WordLayer::Sym2WordLayer(Symbols symbols) : symbols_(std::move(symbols)) {
+  for (const auto& [symbol, readings] : symbols_) {
+    keys_.push_back(symbol);
+  }
+}
+
+void Sym2WordLayer::apply(Document& document) const {
+  std::u32string_view text = document.text();
+
+  for (size_t start = 0; start < text.size();) {
+    // 1. A run of symbols, with the whitespaces around it: " & ", "++"
+    size_t symbol = start;
+    while (symbol < text.size() && unicode::isspace(text[symbol])) {
+      symbol++;
+    }
+    std::vector<std::u32string_view> symbols;
+    size_t end = symbol;
+    for (auto keys = keys_at(text, end, keys_); !keys.empty(); keys = keys_at(text, end, keys_)) {
+      symbols.push_back(keys.front());
+      end += keys.front().size();
+    }
+    if (symbols.empty()) {
+      start = symbol > start ? symbol : start + 1;
+      continue;
+    }
+    while (end < text.size() && unicode::isspace(text[end])) {
+      end++;
+    }
+
+    // 2. The words of each symbol in the context of the run
+    Context context(text, {start, end});
+    std::vector<std::u32string> words;
+    for (auto s : symbols) {
+      auto reading = read(symbols_.at(std::u32string(s)), context);
+      if (std::u32string word = reading.value_or(std::u32string(s)); !word.empty()) {
+        words.push_back(word);
+      }
+    }
+
+    // 3. The words, separated from the text around by single spaces (none before punctuation).
+    // A run right after another one already has its space: "™ &" -> "trademark and"
+    bool left = start > 0 && !unicode::isspace(text[start - 1]);
+    std::u32string replacement;
+    if (words.empty()) {
+      replacement = left && end < text.size() ? U" " : U"";
+    } else {
+      bool right = end < text.size() && !strings::contains(constants::kPunctuation, text[end]);
+      replacement = (left ? U" " : U"") + join(words) + (right ? U" " : U"");
+    }
+    document.replace({start, end}, replacement);
+    start = end;
+  }
+}
+
+} // namespace phonemis::processor::sym2word

package/third-party/common/phonemis/src/phonemis/base/processor/layers/sym2word/layer.h

@@ -0,0 +1,29 @@
+#pragma once
+
+#include "../../common/conditions.h"
+#include "../../layer.h"
+
+#include <string>
+#include <vector>
+
+namespace phonemis::processor::sym2word {
+
+// Symbols with their readings: & -> and; + -> plus after a name (Google+), más otherwise.
+using Symbols = Readings;
+
+/**
+ * Reads the symbols left in a text as words, after numbers are read.
+ * A symbol whose readings all have unmet conditions stays as written.
+ */
+class Sym2WordLayer : public Layer {
+public:
+  explicit Sym2WordLayer(Symbols symbols);
+
+  void apply(Document& document) const override;
+
+private:
+  Symbols symbols_;
+  Keys keys_;
+};
+
+} // namespace phonemis::processor::sym2word