react-native-executorch 0.10.6 → 0.10.7

package/third-party/common/phonemis/src/phonemis/base/pipeline.cpp

@@ -59,6 +59,14 @@ std::u32string Pipeline::operator()(std::u32string_view text, bool preprocess,
   return (*impl_)(text, preprocess, postprocess);
 }
 
+PhonemizedText Pipeline::phonemize_words(std::string_view text) {
+  return impl_->phonemize_words(text);
+}
+
+PhonemizedText Pipeline::phonemize_words(std::u32string_view text) {
+  return impl_->phonemize_words(text);
+}
+
 std::u32string Pipeline::preprocess(const std::u32string& input) {
   return impl_->preprocess(input);
 }

package/third-party/common/phonemis/src/phonemis/base/pipeline.h

@@ -35,6 +35,25 @@ public:
                             bool preprocess = true,
                             bool postprocess = true) override;
 
+  /**
+   * Performs a complete G2P processing and locates each word of the input in
+   * the result. Every word maps onto its own space-separated phoneme group,
+   * except for numbers, which take as many groups as they were spelled out into.
+   * @param text an input text (utf-8) to be processed.
+   * @returns phonemization of given input text, with its words located in it.
+   * Word offsets count code points of the input converted to u32.
+   */
+  PhonemizedText phonemize_words(std::string_view text) override;
+
+  /**
+   * Performs a complete G2P processing and locates each word of the input in
+   * the result. Every word maps onto its own space-separated phoneme group,
+   * except for numbers, which take as many groups as they were spelled out into.
+   * @param text an input text (u32string_view) to be processed.
+   * @returns phonemization of given input text, with its words located in it.
+   */
+  PhonemizedText phonemize_words(std::u32string_view text) override;
+
   // Performs a preprocessing stage of the pipeline.
   std::u32string preprocess(const std::u32string& input) override;
 

package/third-party/common/phonemis/src/phonemis/base/processor/common/conditions.cpp

@@ -0,0 +1,37 @@
+#include "conditions.h"
+#include "text.h"
+#include <phonemis/utils/unicode.h>
+
+namespace phonemis::processor {
+
+using namespace utils;
+
+std::optional<std::u32string> read(const std::vector<Reading>& readings, const Context& context) {
+  for (const auto& reading : readings) {
+    if (!reading.when || reading.when(context)) {
+      return reading.words;
+    }
+  }
+  return std::nullopt;
+}
+
+bool after_name(const Context& context) {
+  std::u32string_view head = context.head();
+  return head.size() >= 2 && unicode::isalpha(head[head.size() - 1]) && unicode::isalpha(head[head.size() - 2]);
+}
+
+bool before_number(const Context& context) {
+  std::u32string_view tail = context.tail();
+  return is_digit_after(tail, skip_space(tail, 0));
+}
+
+bool before_name(const Context& context) {
+  std::u32string word = context.after();
+  return !word.empty() && unicode::isupper(word[0]);
+}
+
+Condition after_any(std::unordered_set<std::u32string> words) {
+  return [words = std::move(words)](const Context& context) { return words.contains(context.before()); };
+}
+
+} // namespace phonemis::processor

package/third-party/common/phonemis/src/phonemis/base/processor/common/conditions.h

@@ -0,0 +1,45 @@
+#pragma once
+
+#include "context.h"
+
+#include <functional>
+#include <optional>
+#include <string>
+#include <unordered_map>
+#include <unordered_set>
+#include <vector>
+
+namespace phonemis::processor {
+
+// A condition on the context of a written phrase, which decides between its readings.
+using Condition = std::function<bool(const Context&)>;
+
+// A way of saying a written phrase (a symbol, an abbreviation), when its condition holds.
+struct Reading {
+  Reading(const char32_t* words, Condition when = nullptr) : words(words), when(std::move(when)) {}
+
+  std::u32string words; // Empty words drop the phrase
+  Condition when;       // Always, if none
+};
+
+// Written phrases with their readings: the first one whose condition holds is used.
+using Readings = std::unordered_map<std::u32string, std::vector<Reading>>;
+
+/**
+ * The words of a phrase in its context.
+ * @returns the words of its first reading whose condition holds, or std::nullopt if there is none.
+ */
+std::optional<std::u32string> read(const std::vector<Reading>& readings, const Context& context);
+
+// -----------------------
+// ----- Conditions -----
+// -----------------------
+
+bool after_name(const Context& context);    // Two letters right before: Google+
+bool before_number(const Context& context); // A digit right after, possibly after a space: nr 5, nr5
+bool before_name(const Context& context);   // A capitalized word after: St. Louis
+
+// The word before is one of given words: im § 5
+Condition after_any(std::unordered_set<std::u32string> words);
+
+} // namespace phonemis::processor

package/third-party/common/phonemis/src/phonemis/base/processor/common/constants.h

@@ -0,0 +1,10 @@
+#pragma once
+
+#include <string_view>
+
+namespace phonemis::processor::constants {
+
+// Punctuation stripped from the words around a match
+inline constexpr std::u32string_view kPunctuation = U".,;:!?\"'()[]«»„“”…";
+
+} // namespace phonemis::processor::constants

package/third-party/common/phonemis/src/phonemis/base/processor/common/context.cpp

@@ -0,0 +1,105 @@
+#include "constants.h"
+#include "context.h"
+#include <phonemis/utils/strings.h>
+#include <phonemis/utils/unicode.h>
+
+namespace phonemis::processor {
+
+using namespace utils;
+
+Context::Context(std::u32string_view text, Span span) : text_(text), span_(span) {}
+
+std::u32string Context::before(size_t k) const {
+  // Walk back over k words of the head
+  std::u32string_view head = this->head();
+  size_t end = head.size();
+  for (; k > 0; --k) {
+    while (end > 0 && unicode::isspace(head[end - 1])) {
+      end--;
+    }
+    if (end == 0) {
+      return U"";
+    }
+    size_t begin = end;
+    while (begin > 0 && !unicode::isspace(head[begin - 1])) {
+      begin--;
+    }
+    if (k == 1) {
+      return strings::to_lower(stripped(head.substr(begin, end - begin)));
+    }
+    end = begin;
+  }
+
+  return U"";
+}
+
+std::u32string Context::after(size_t k) const {
+  std::u32string_view tail = this->tail();
+  if (!tail.empty() && strings::contains(constants::kPunctuation, tail[0])) {
+    return U"";
+  }
+
+  // Walk forward over k words of the tail
+  size_t begin = 0;
+  for (; k > 0; --k) {
+    while (begin < tail.size() && unicode::isspace(tail[begin])) {
+      begin++;
+    }
+    if (begin == tail.size()) {
+      return U"";
+    }
+    size_t end = begin;
+    while (end < tail.size() && !unicode::isspace(tail[end])) {
+      end++;
+    }
+    if (k == 1) {
+      return std::u32string(stripped(tail.substr(begin, end - begin)));
+    }
+    begin = end;
+  }
+  return U"";
+}
+
+std::u32string_view Context::head() const {
+  return text_.substr(0, span_.begin);
+}
+
+std::u32string_view Context::tail() const {
+  return text_.substr(span_.end);
+}
+
+std::u32string_view Context::text() const {
+  return text_.substr(span_.begin, span_.end - span_.begin);
+}
+
+std::vector<std::u32string_view> words_of(std::u32string_view text) {
+  std::vector<std::u32string_view> words;
+
+  for (size_t i = 0; i < text.size();) {
+    if (unicode::isspace(text[i])) {
+      i++;
+      continue;
+    }
+    size_t end = i;
+    while (end < text.size() && !unicode::isspace(text[end])) {
+      end++;
+    }
+    words.push_back(text.substr(i, end - i));
+    i = end;
+  }
+
+  return words;
+}
+
+std::u32string_view stripped(std::u32string_view word) {
+  size_t begin = word.find_first_not_of(constants::kPunctuation);
+  if (begin == std::u32string_view::npos) {
+    return U"";
+  }
+
+  size_t end = word.find_last_not_of(constants::kPunctuation);
+
+  return word.substr(begin, end - begin + 1);
+}
+
+} // namespace phonemis::processor

package/third-party/common/phonemis/src/phonemis/base/processor/common/context.h

@@ -0,0 +1,49 @@
+#pragma once
+
+#include "types.h"
+
+#include <string>
+#include <string_view>
+#include <vector>
+
+namespace phonemis::processor {
+
+/**
+ * The surroundings of a span of a text: the words before and after it.
+ * 
+ * A word the span starts or ends inside of counts with its part outside the span.
+ */
+class Context {
+public:
+  Context(std::u32string_view text, Span span);
+
+  /**
+   * The k-th word before the span, lowercased and without surrounding punctuation.
+   * Empty if there is no such word.
+   */
+  std::u32string before(size_t k = 1) const;
+
+  /**
+   * The k-th word after the span, case kept, without surrounding punctuation.
+   * Empty if there is no such word, or if a punctuation mark directly follows the span.
+   */
+  std::u32string after(size_t k = 1) const;
+
+  std::u32string_view head() const; // The text before the span
+  std::u32string_view tail() const; // The text after the span
+  std::u32string_view text() const; // The span itself
+
+  Span span() const { return span_; }
+
+private:
+  std::u32string_view text_;
+  Span span_;
+};
+
+// Whitespace-separated words of a text, like Python's str.split().
+std::vector<std::u32string_view> words_of(std::u32string_view text);
+
+// A word without the punctuation around it.
+std::u32string_view stripped(std::u32string_view word);
+
+} // namespace phonemis::processor

package/third-party/common/phonemis/src/phonemis/base/processor/common/document.cpp

@@ -0,0 +1,173 @@
+#include "document.h"
+
+#include <algorithm>
+#include <stdexcept>
+
+namespace phonemis::processor {
+
+Document::Document(std::u32string_view text) : text_(text), original_size_(text.size()) {
+  // At first, the whole text is a single copied piece
+  if (!text.empty()) {
+    pieces_.push_back({.original = {0, text.size()}, .size = text.size()});
+  }
+}
+
+void Document::replace(Span span, std::u32string replacement) {
+  if (span.begin > span.end || span.end > text_.size()) {
+    throw std::invalid_argument("Document: edit span out of range");
+  }
+
+  // Keep the edits sorted, so that commit() applies them in a single pass
+  auto it = std::ranges::upper_bound(edits_, span.begin, {}, [](const Edit& edit) { return edit.span.begin; });
+  bool overlaps_previous = it != edits_.begin() && std::prev(it)->span.end > span.begin;
+  bool overlaps_next = it != edits_.end() && it->span.begin < span.end;
+  if (overlaps_previous || overlaps_next) {
+    throw std::invalid_argument("Document: overlapping edits");
+  }
+
+  edits_.insert(it, {span, std::move(replacement)});
+}
+
+void Document::commit() {
+  if (edits_.empty()) {
+    return;
+  }
+
+  // Where each piece starts in the current text
+  std::vector<size_t> starts;
+  for (size_t start = 0; const auto& piece : pieces_) {
+    starts.push_back(start);
+    start += piece.size;
+  }
+
+  // 1. Group the edits into the spans they really change.
+  // An edit inside an earlier replacement replaces all of it ("ten" -> "Ten" changes the whole "ten" of "10"),
+  // and edits whose widened spans meet are merged: each group (an edit itself) becomes one replaced piece.
+  std::vector<Edit> groups;
+  for (size_t i = 0; i < edits_.size();) {
+    // Extend the group while the next widened edit starts inside it
+    Edit group{.span = widen(edits_[i].span, starts)};
+    size_t last = i;
+    while (last + 1 < edits_.size() && widen(edits_[last + 1].span, starts).begin < group.span.end) {
+      group.span.end = std::max(group.span.end, widen(edits_[++last].span, starts).end);
+    }
+
+    // Its text: the current text of the span, with each edit's text in place of its range
+    size_t position = group.span.begin;
+    for (; i <= last; ++i) {
+      group.text.append(text_, position, edits_[i].span.begin - position);
+      group.text.append(edits_[i].text);
+      position = edits_[i].span.end;
+    }
+    group.text.append(text_, position, group.span.end - position);
+    groups.push_back(std::move(group));
+  }
+
+  // 2. Rebuild the text and its pieces, walking the groups and the old pieces side by side.
+  // Pieces before a group are kept, pieces it touches are merged into one replaced piece.
+  std::u32string text;
+  std::vector<Piece> pieces;
+  std::vector<Piece> old = pieces_;
+  size_t next = 0;     // The first old piece not handled yet
+  size_t position = 0; // The end of the current text copied so far
+
+  for (const auto& [span, replacement] : groups) {
+    // An empty piece (a deletion) counts as touched when it lies at the group's edge, so it merges into the group
+    auto end_of = [&](size_t i) { return starts[i] + old[i].size; };
+    auto before = [&](size_t i) { return old[i].size > 0 ? end_of(i) <= span.begin : starts[i] < span.begin; };
+    auto touched = [&](size_t i) {
+      return old[i].size > 0 ? starts[i] < span.end && span.begin < end_of(i) :
+                               span.begin <= starts[i] && starts[i] <= span.end;
+    };
+
+    // 2.1. The pieces before the group stay as they are
+    while (next < old.size() && before(next)) {
+      pieces.push_back(old[next++]);
+    }
+
+    // 2.2. The original span behind the group: from its first touched piece to its last one.
+    // An insertion between pieces touches none, and has an empty original span.
+    size_t original_begin = next < old.size() ? old[next].original.begin : original_size_;
+    size_t original_end = original_begin;
+    if (next < old.size() && touched(next)) {
+      // A copied piece starting before the group keeps its head: "I have 2 cats" -> "I have " stays copied
+      if (!old[next].replaced && starts[next] < span.begin) {
+        size_t cut = span.begin - starts[next];
+        pieces.push_back({.original = {original_begin, original_begin + cut}, .size = cut});
+        original_begin += cut;
+      }
+
+      size_t last = next;
+      while (next < old.size() && touched(next)) {
+        last = next++;
+      }
+      original_end = old[last].original.end;
+
+      // A copied piece ending after the group keeps its tail as an old piece: the next group may cut it again
+      if (!old[last].replaced && end_of(last) > span.end) {
+        size_t cut = span.end - starts[last];
+        original_end = old[last].original.begin + cut;
+        old[last].original.begin += cut;
+        old[last].size -= cut;
+        starts[last] = span.end;
+        next = last;
+      }
+    }
+
+    // 2.3. The text up to the group, then the group's replacement as one replaced piece
+    text.append(text_, position, span.begin - position);
+    text.append(replacement);
+    pieces.push_back({.original = {original_begin, original_end}, .size = replacement.size(), .replaced = true});
+    position = span.end;
+  }
+
+  // 3. The rest of the text after the last group, and its pieces
+  text.append(text_, position);
+  pieces.insert(pieces.end(), old.begin() + next, old.end());
+
+  text_ = std::move(text);
+  pieces_ = std::move(pieces);
+  edits_.clear();
+}
+
+std::vector<Document::Edit> Document::changes() const {
+  std::vector<Edit> changes;
+
+  // Walk the pieces with their position in the current text; only the replaced ones are changes
+  size_t position = 0;
+  bool previous_replaced = false;
+  for (const auto& piece : pieces_) {
+    if (piece.replaced) {
+      // Adjacent replaced pieces form a single change
+      if (previous_replaced) {
+        changes.back().span.end = piece.original.end;
+        changes.back().text.append(text_, position, piece.size);
+      } else {
+        changes.push_back({piece.original, text_.substr(position, piece.size)});
+      }
+    }
+    previous_replaced = piece.replaced;
+    position += piece.size;
+  }
+  return changes;
+}
+
+Span Document::widen(Span span, std::span<const size_t> starts) const {
+  // An edge falling strictly inside a replaced piece moves to that piece's edge
+  for (size_t i = 0; i < pieces_.size(); ++i) {
+    size_t begin = starts[i], end = begin + pieces_[i].size;
+    if (!pieces_[i].replaced) {
+      continue;
+    }
+    if (begin < span.begin && span.begin < end) {
+      span.begin = begin;
+    }
+    if (begin < span.end && span.end < end) {
+      span.end = end;
+    }
+  }
+
+  return span;
+}
+
+} // namespace phonemis::processor

package/third-party/common/phonemis/src/phonemis/base/processor/common/document.h

@@ -0,0 +1,67 @@
+#pragma once
+
+#include "types.h"
+
+#include <span>
+#include <string>
+#include <string_view>
+#include <vector>
+
+namespace phonemis::processor {
+
+/**
+ * The text being preprocessed, with lazy edits.
+ * 
+ * Provides an abstraction which allows algorithms like Num2Word patterns operate freely, without
+ * risking that some previous edit affects the future one or the algorithm.
+ */
+class Document {
+public:
+  // A span of a text, with the text in its place.
+  struct Edit {
+    Span span;
+    std::u32string text;
+  };
+
+  // A piece of the current text: a copy of a span of the original text, or the text which replaced it.
+  // A replaced piece maps to its original span only as a whole.
+  struct Piece {
+    Span original;
+    size_t size = 0; // In the current text
+    bool replaced = false;
+  };
+
+  explicit Document(std::u32string_view text);
+
+  std::u32string_view text() const { return text_; }
+
+  /**
+   * Queues a replacement of a span of the current text.
+   * @param span a range of the text as it was when the current layer started.
+   * @param replacement the new content of that range.
+   * @throws std::invalid_argument if the span is out of range or overlaps an already queued edit.
+   */
+  void replace(Span span, std::u32string replacement);
+
+  /**
+   * Applies all queued edits at once.
+   */
+  void commit();
+
+  /**
+   * The committed changes of the original text, in order: each replaced span of the original text with
+   * the current text in its place.
+   */
+  std::vector<Edit> changes() const;
+
+private:
+  // The span of the current text an edit really changes: widened to whole replaced pieces it touches.
+  Span widen(Span span, std::span<const size_t> starts) const;
+
+  std::u32string text_;
+  std::vector<Edit> edits_; // Sorted by position
+  std::vector<Piece> pieces_;
+  size_t original_size_;
+};
+
+} // namespace phonemis::processor

package/third-party/common/phonemis/src/phonemis/base/processor/common/text.cpp

@@ -0,0 +1,75 @@
+#include "text.h"
+#include <phonemis/utils/strings.h>
+#include <phonemis/utils/unicode.h>
+
+#include <algorithm>
+
+namespace phonemis::processor {
+
+using namespace utils;
+
+bool is_clear_before(std::u32string_view text, size_t position, std::u32string_view also) {
+  return position == 0 || (!unicode::isword(text[position - 1]) && !strings::contains(also, text[position - 1]));
+}
+
+bool is_clear_after(std::u32string_view text, size_t position, std::u32string_view also) {
+  return position >= text.size() || (!unicode::isword(text[position]) && !strings::contains(also, text[position]));
+}
+
+bool is_digit_after(std::u32string_view text, size_t position, std::u32string_view marks) {
+  if (position < text.size() && strings::contains(marks, text[position])) {
+    position++;
+  }
+
+  return position < text.size() && unicode::isdigit(text[position]);
+}
+
+bool is_sentence_end(std::u32string_view text, size_t position, bool capital) {
+  while (position < text.size() && unicode::isspace(text[position])) {
+    position++;
+  }
+
+  return position == text.size() || (capital && unicode::isupper(text[position]));
+}
+
+size_t digits_end(std::u32string_view text, size_t position) {
+  while (position < text.size() && unicode::isdigit(text[position])) {
+    position++;
+  }
+
+  return position;
+}
+
+size_t skip_space(std::u32string_view text, size_t position) {
+  return position < text.size() && unicode::isspace(text[position]) ? position + 1 : position;
+}
+
+std::vector<std::u32string_view> keys_at(std::u32string_view text, size_t position, const Keys& keys) {
+  std::vector<std::u32string_view> found;
+  for (const auto& key : keys) {
+    if (text.substr(position).starts_with(key)) {
+      found.push_back(key);
+    }
+  }
+
+  std::ranges::sort(found, [](auto a, auto b) { return a.size() > b.size(); });
+
+  return found;
+}
+
+std::u32string join(std::span<const std::u32string> words, std::u32string_view separator) {
+  std::u32string result;
+  for (const auto& word : words) {
+    if (word.empty()) {
+      continue;
+    }
+    if (!result.empty()) {
+      result += separator;
+    }
+    result += word;
+  }
+
+  return result;
+}
+
+} // namespace phonemis::processor

package/third-party/common/phonemis/src/phonemis/base/processor/common/text.h

@@ -0,0 +1,60 @@
+#pragma once
+
+#include "types.h"
+
+#include <span>
+#include <string>
+#include <string_view>
+#include <vector>
+
+/**
+ * Scanning primitives shared by the layers: lookarounds and table keys at a position of a text.
+ * Character classes come from utils/unicode.h.
+ */
+namespace phonemis::processor {
+
+// -----------------------
+// ----- Lookarounds -----
+// -----------------------
+
+/**
+ * Checks if a position starts a word: no word character stands right before it.
+ * @param also other characters which may not stand there, e.g. "." to reject the "g" of "e.g".
+ */
+bool is_clear_before(std::u32string_view text, size_t position, std::u32string_view also = U"");
+
+/**
+ * Checks if a position ends a word: no word character stands at it, or the text ends there.
+ * @param also other characters which may not stand there.
+ */
+bool is_clear_after(std::u32string_view text, size_t position, std::u32string_view also = U"");
+
+/**
+ * Checks if a digit stands at a position, possibly after one of given marks: "5", ".5".
+ * @param marks characters which may stand between the position and the digit.
+ */
+bool is_digit_after(std::u32string_view text, size_t position, std::u32string_view marks = U"");
+
+/**
+ * Checks if a sentence ends at a position: only whitespaces follow it.
+ * @param capital whether a capital letter after the whitespaces also starts a new sentence: "2020 r. Potem".
+ */
+bool is_sentence_end(std::u32string_view text, size_t position, bool capital = false);
+
+// ---------------------------
+// ----- Runs and keys -----
+// ---------------------------
+
+// End of the run of digits starting at a position.
+size_t digits_end(std::u32string_view text, size_t position);
+
+// Position after an optional whitespace.
+size_t skip_space(std::u32string_view text, size_t position);
+
+// The keys starting at a position, longest first.
+std::vector<std::u32string_view> keys_at(std::u32string_view text, size_t position, const Keys& keys);
+
+// The non-empty words, joined with a separator.
+std::u32string join(std::span<const std::u32string> words, std::u32string_view separator = U" ");
+
+} // namespace phonemis::processor

package/third-party/common/phonemis/src/phonemis/base/processor/common/types.h

@@ -0,0 +1,18 @@
+#pragma once
+
+#include <cstddef>
+#include <string>
+#include <vector>
+
+namespace phonemis::processor {
+
+// A half-open range [begin, end) of characters in a document's text.
+struct Span {
+  size_t begin = 0;
+  size_t end = 0;
+};
+
+// Keys of a table matched in text (units, currencies, abbreviations); the longest matching one wins.
+using Keys = std::vector<std::u32string>;
+
+} // namespace phonemis::processor