scriptconv 0.0.4a35__tar.gz → 0.0.4a37__tar.gz

{scriptconv-0.0.4a35/scriptconv.egg-info → scriptconv-0.0.4a37}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: scriptconv
-Version: 0.0.4a35
+Version: 0.0.4a37
 Summary: Zero-dependency script & phoneme-notation core — ISO-15924 detection & metadata, IPA↔ARPABET/X-SAMPA/Lexique/Kirshenbaum/Cotovía/RFE, Buckwalter↔Arabic, Hangul→jamo, kana
 License: Apache-2.0
 Project-URL: Homepage, https://github.com/TigreGotico/scriptconv
@@ -11,13 +11,12 @@ Classifier: Intended Audience :: Developers
 Classifier: Intended Audience :: Science/Research
 Classifier: Topic :: Text Processing :: Linguistic
 Classifier: Programming Language :: Python :: 3
-Classifier: Programming Language :: Python :: 3.10
 Classifier: Programming Language :: Python :: 3.11
 Classifier: Programming Language :: Python :: 3.12
 Classifier: Programming Language :: Python :: 3.13
 Classifier: License :: OSI Approved :: Apache Software License
 Classifier: Operating System :: OS Independent
-Requires-Python: >=3.10
+Requires-Python: >=3.11
 Description-Content-Type: text/markdown
 License-File: LICENSE
 Provides-Extra: ja
@@ -54,7 +53,8 @@ Requires-Dist: onnxruntime; extra == "byt5"
 Requires-Dist: numpy; extra == "byt5"
 Provides-Extra: o2i
 Requires-Dist: scriptconv[phonemizers]; extra == "o2i"
-Requires-Dist: orthography2ipa; extra == "o2i"
+Requires-Dist: orthography2ipa>=7.112.0a1; extra == "o2i"
+Requires-Dist: o2i-lexicons>=0.1.0a1; extra == "o2i"
 Provides-Extra: eu
 Requires-Dist: scriptconv[phonemizers]; extra == "eu"
 Requires-Dist: euskaphone; extra == "eu"
@@ -133,6 +133,9 @@ Requires-Dist: bifonia>=0.1.1; extra == "test"
 Requires-Dist: stressonnx>=0.0.3a2; extra == "test"
 Requires-Dist: tugaphone>=1.2.0a1; extra == "test"
 Requires-Dist: ghana-g2p>=0.1.1; extra == "test"
+Requires-Dist: espyak; extra == "test"
+Requires-Dist: orthography2ipa>=7.112.0a1; extra == "test"
+Requires-Dist: o2i-lexicons>=0.1.0a1; extra == "test"
 Requires-Dist: tomli; python_version < "3.11" and extra == "test"
 Dynamic: license-file
 
@@ -390,7 +393,7 @@ g.convert("bom dia", "text", "ipa", lang="pt")     # 'ˈbõ ˈdʒiɐ'
 g.can_convert("text", "arpa")                      # True (chains through IPA)
 ```
 
-Two design points worth knowing:
+Three design points worth knowing:
 
 - **Normalization is injectable.** TTS stacks expand numbers and dates
   before phonemizing, and that needs language resources scriptconv does
@@ -405,6 +408,17 @@ Two design points worth knowing:
   overrides it with a path or callable; the cache location is set via
   `SCRIPTCONV_CACHE` or `XDG_CACHE_HOME`).
 
+- **The orthography2ipa phonemizer uses the packaged lexicons.** The `o2i`
+  extra installs [o2i-lexicons](https://github.com/TigreGotico/o2i-lexicons),
+  which holds the words that the orthography2ipa rules get wrong, such as
+  en-GB ⟨yacht⟩. The key `packaged_lexicons` (default `True`) enables the
+  packaged lexicon for each language that the phonemizer transcribes, when a
+  package serves that language. Set it to `False` to use the rules only:
+  `get_phonemizer(Phonemizer.ORTHOGRAPHY2IPA, packaged_lexicons=False)`.
+  orthography2ipa keeps its lexicons for the whole process. After one caller
+  enables a lexicon, every orthography2ipa caller in the process uses it, and
+  `packaged_lexicons=False` does not disable it again.
+
 **Pre-G2P disambiguation.** `add_diacritics(text, lang, model=None)`
 restores information ordinary orthography omits but a G2P needs, before
 phonemization. It covers Hebrew niqqud (phonikud), Arabic tashkeel

{scriptconv-0.0.4a35 → scriptconv-0.0.4a37}/README.md

@@ -252,7 +252,7 @@ g.convert("bom dia", "text", "ipa", lang="pt")     # 'ˈbõ ˈdʒiɐ'
 g.can_convert("text", "arpa")                      # True (chains through IPA)
 ```
 
-Two design points worth knowing:
+Three design points worth knowing:
 
 - **Normalization is injectable.** TTS stacks expand numbers and dates
   before phonemizing, and that needs language resources scriptconv does
@@ -267,6 +267,17 @@ Two design points worth knowing:
   overrides it with a path or callable; the cache location is set via
   `SCRIPTCONV_CACHE` or `XDG_CACHE_HOME`).
 
+- **The orthography2ipa phonemizer uses the packaged lexicons.** The `o2i`
+  extra installs [o2i-lexicons](https://github.com/TigreGotico/o2i-lexicons),
+  which holds the words that the orthography2ipa rules get wrong, such as
+  en-GB ⟨yacht⟩. The key `packaged_lexicons` (default `True`) enables the
+  packaged lexicon for each language that the phonemizer transcribes, when a
+  package serves that language. Set it to `False` to use the rules only:
+  `get_phonemizer(Phonemizer.ORTHOGRAPHY2IPA, packaged_lexicons=False)`.
+  orthography2ipa keeps its lexicons for the whole process. After one caller
+  enables a lexicon, every orthography2ipa caller in the process uses it, and
+  `packaged_lexicons=False` does not disable it again.
+
 **Pre-G2P disambiguation.** `add_diacritics(text, lang, model=None)`
 restores information ordinary orthography omits but a G2P needs, before
 phonemization. It covers Hebrew niqqud (phonikud), Arabic tashkeel

{scriptconv-0.0.4a35 → scriptconv-0.0.4a37}/pyproject.toml

@@ -8,14 +8,13 @@ dynamic = ["version", "dependencies"]
 description = "Zero-dependency script & phoneme-notation core — ISO-15924 detection & metadata, IPA↔ARPABET/X-SAMPA/Lexique/Kirshenbaum/Cotovía/RFE, Buckwalter↔Arabic, Hangul→jamo, kana"
 readme = "README.md"
 license = {text = "Apache-2.0"}
-requires-python = ">=3.10"
+requires-python = ">=3.11"
 classifiers = [
     "Development Status :: 3 - Alpha",
     "Intended Audience :: Developers",
     "Intended Audience :: Science/Research",
     "Topic :: Text Processing :: Linguistic",
     "Programming Language :: Python :: 3",
-    "Programming Language :: Python :: 3.10",
     "Programming Language :: Python :: 3.11",
     "Programming Language :: Python :: 3.12",
     "Programming Language :: Python :: 3.13",
@@ -45,7 +44,7 @@ transphone = ["scriptconv[phonemizers]", "transphone"]
 phonetisaurus = ["scriptconv[phonemizers]", "phonetisaurus"]
 misaki = ["scriptconv[phonemizers]", "misaki"]
 byt5 = ["scriptconv[phonemizers]", "onnxruntime", "numpy"]
-o2i = ["scriptconv[phonemizers]", "orthography2ipa"]
+o2i = ["scriptconv[phonemizers]", "orthography2ipa>=7.112.0a1", "o2i-lexicons>=0.1.0a1"]
 eu = ["scriptconv[phonemizers]", "euskaphone", "ahotts_g2p"]
 pt-phonemizers = ["scriptconv[phonemizers]", "tugaphone>=1.2.0a1", "g2p_barranquenho"]
 mwl = ["scriptconv[phonemizers]", "mwl_phonemizer"]
@@ -63,7 +62,7 @@ zh-phonemizers = ["scriptconv[phonemizers]", "jieba", "pypinyin>=0.50,<1", "xpin
 en-phonemizers = ["scriptconv[phonemizers]", "g2p_en", "openphonemizer", "deep-phonemizer", "requests"]
 ar-phonemizers = ["scriptconv[phonemizers]", "arbtok", "ovos-number-parser"]
 shami = ["scriptconv[phonemizers]", "regex"]
-test = ["pytest", "pytest-timeout", "pykakasi>=2.3,<3", "pypinyin>=0.50,<1", "quebra-frases", "langcodes", "regex", "ovos-number-parser", "bifonia>=0.1.1", "stressonnx>=0.0.3a2", "tugaphone>=1.2.0a1", "ghana-g2p>=0.1.1", "tomli; python_version < '3.11'"]
+test = ["pytest", "pytest-timeout", "pykakasi>=2.3,<3", "pypinyin>=0.50,<1", "quebra-frases", "langcodes", "regex", "ovos-number-parser", "bifonia>=0.1.1", "stressonnx>=0.0.3a2", "tugaphone>=1.2.0a1", "ghana-g2p>=0.1.1", "espyak", "orthography2ipa>=7.112.0a1", "o2i-lexicons>=0.1.0a1", "tomli; python_version < '3.11'"]
 
 [project.urls]
 Homepage = "https://github.com/TigreGotico/scriptconv"

{scriptconv-0.0.4a35 → scriptconv-0.0.4a37}/scriptconv/phonemizers/base.py

@@ -34,6 +34,52 @@ RawPhonemizedChunks = List[Tuple[str, str, bool]]
 
 PhonemizedChunks = list[list[str]]
 
+# Clause punctuation kept by ``keep_punctuation``, mapped to the symbol a voice
+# is trained on. These are the six clause types Piper appends after every
+# espeak-ng clause (piper-phonemize ``src/phonemize.cpp``): a full stop,
+# question mark and exclamation mark stand alone, a comma, colon and semicolon
+# are followed by a space. Arabic script has its own comma, semicolon and
+# question mark; they map onto the same symbols.
+CLAUSE_PUNCTUATION = {".": ".", "?": "?", "!": "!", ",": ",", ":": ":", ";": ";",
+                      "\u060c": ",", "\u061b": ";", "\u061f": "?"}
+_PAUSE_PUNCTUATION = {",", ":", ";"}
+# A clause ends at punctuation followed by whitespace, as in espeak-ng, so the
+# "," in "10,4" or the ":" in "16:30" never splits. A lone "." is left to the
+# sentence tokenizer, which knows abbreviations; inside a sentence an ellipsis
+# or "|" still ends a clause, as in chunk_text, but carries no symbol (espeak-ng
+# reads an ellipsis as a spoken-name pause, which Piper does not keep).
+_CLAUSE_BREAK = re.compile(r"(?:\.\.+|\u2026|[,:;!?|\u060c\u061b\u061f])[^\w\s]*(?=\s)")
+_TRAILING_PUNCTUATION = re.compile(r"[^\w\s]+\s*$")
+_LANG_FLAGS = re.compile(r"\([^)]+\)")
+
+
+def _clause_mark(punctuation: str) -> Optional[str]:
+    """The symbol a run of clause-final punctuation is kept as, if any."""
+    if ".." in punctuation or "\u2026" in punctuation:
+        return None
+    for char in punctuation:
+        if char in CLAUSE_PUNCTUATION:
+            return CLAUSE_PUNCTUATION[char]
+    return None
+
+
+def split_clauses(sentence: str) -> List[Tuple[str, Optional[str]]]:
+    """Split one sentence into ``(clause, mark)`` pairs.
+
+    ``mark`` is the symbol from :data:`CLAUSE_PUNCTUATION` that ends the
+    clause, or ``None`` when the clause ends without one (no punctuation, an
+    ellipsis, ``|``). The clause text still carries its own punctuation.
+    """
+    clauses: List[Tuple[str, Optional[str]]] = []
+    start = 0
+    for match in _CLAUSE_BREAK.finditer(sentence):
+        clauses.append((sentence[start:match.start()], _clause_mark(match.group())))
+        start = match.end()
+    tail = sentence[start:]
+    trailing = _TRAILING_PUNCTUATION.search(tail)
+    clauses.append((tail, _clause_mark(trailing.group()) if trailing else None))
+    return clauses
+
 
 class MissingLanguageError(ValueError):
     """No language reached :meth:`BasePhonemizer.match_lang` at all.
@@ -75,6 +121,18 @@ def _check_alphabet(phonemizer, alphabet: Alphabet,
 
 
 class BasePhonemizer(metaclass=abc.ABCMeta):
+    keep_punctuation: bool = False
+    """Keep clause punctuation (``, . ? ! : ;``) in the phoneme output.
+
+    Off by default, so voices trained without punctuation receive exactly the
+    sequences they were trained on. When on, :meth:`phonemize_to_list` and
+    :meth:`phonemize_lazy` phonemize clause by clause and append each clause's
+    symbol the way Piper does, and the two return the same phonemes for the
+    same text. A backend that overrides either method owns its own output and
+    ignores this flag, since honouring it on one side only would make training
+    and synthesis disagree.
+    """
+
     def __init__(self, alphabet: Alphabet = Alphabet.UNICODE,
                  normalizer: Optional[Callable[[str, str], str]] = None):
         super().__init__()
@@ -88,7 +146,17 @@ class BasePhonemizer(metaclass=abc.ABCMeta):
         raise NotImplementedError
 
     def phonemize_to_list(self, text: str, lang: str) -> List[str]:
-        return list(self.phonemize_string(text, lang))
+        """Phonemes of *text* as one flat list, without normalization.
+
+        Clause breaks that a backend prints as ``"\\n"`` (espeak-ng does at
+        every comma, question mark and exclamation mark) are not phonemes and
+        are dropped; with :attr:`keep_punctuation` the clause's punctuation
+        takes their place.
+        """
+        if self._keeps_punctuation():
+            return [p for sentence in self._punctuated_sentences(text, lang)
+                    for p in sentence]
+        return [p for p in self.phonemize_string(text, lang) if p != "\n"]
 
     def phonemize(self, text: str, lang: str) -> PhonemizedChunks:
         # PhonemizedChunks is list[list[str]]; empty text yields no
@@ -114,6 +182,9 @@ class BasePhonemizer(metaclass=abc.ABCMeta):
             return
         if self.normalizer is not None:
             text = self.normalizer(text, lang)
+        if self._keeps_punctuation():
+            yield from self._punctuated_sentences(text, lang)
+            return
         for chunk, punct, eos in self.chunk_text(text):
             phoneme_str = self.phonemize_string(self.remove_punctuation(chunk), lang)
             # Filter out (lang) switch (flags) that surround words from languages
@@ -123,6 +194,44 @@ class BasePhonemizer(metaclass=abc.ABCMeta):
             # its own sentence.
             yield list(phoneme_str)
 
+    def _keeps_punctuation(self) -> bool:
+        cls = type(self)
+        return (self.keep_punctuation
+                and cls.phonemize_to_list is BasePhonemizer.phonemize_to_list
+                and cls.phonemize_lazy is BasePhonemizer.phonemize_lazy)
+
+    def _punctuated_sentences(self, text: str, lang: str) -> Iterator[List[str]]:
+        """Per-sentence phonemes with each clause's punctuation appended.
+
+        Each clause is phonemized on its own with its punctuation removed, so
+        the backend sees the same words in training and in synthesis, and the
+        clause's symbol is appended after it: bare for ``. ? !``, followed by a
+        space for ``, : ;``.
+        """
+        try:
+            from quebra_frases import sentence_tokenize
+        except ImportError:
+            raise ImportError(
+                "sentence chunking needs quebra-frases — install with "
+                "`pip install scriptconv[phonemizers]`") from None
+        for sentence in sentence_tokenize(text):
+            phonemes: List[str] = []
+            for clause, mark in split_clauses(sentence):
+                clause = self.remove_punctuation(clause)
+                if not clause:
+                    continue
+                phoneme_str = _LANG_FLAGS.sub("", self.phonemize_string(clause, lang))
+                clause_phonemes = [p for p in phoneme_str if p != "\n"]
+                if not clause_phonemes:
+                    continue
+                phonemes.extend(clause_phonemes)
+                if mark:
+                    phonemes.append(mark)
+                    if mark in _PAUSE_PUNCTUATION:
+                        phonemes.append(" ")
+            if phonemes:
+                yield phonemes
+
     @staticmethod
     def _process_phones(raw_phones: RawPhonemizedChunks) -> PhonemizedChunks:
         """Text to phonemes grouped by sentence."""

{scriptconv-0.0.4a35 → scriptconv-0.0.4a37}/scriptconv/phonemizers/o2ipa.py

@@ -12,6 +12,13 @@ diacritizer on top of this same lattice); for dialect-aware Basque use
 
 ``orthography2ipa`` is imported lazily so importing ``scriptconv`` never requires
 it; install with ``pip install orthography2ipa`` (or ``pip install scriptconv[o2i]``).
+
+With ``packaged_lexicons=True`` (the default) the phonemizer enables the
+installed packaged lexicon (``o2i-lexicons``) for each language it transcribes,
+when a package serves that language. orthography2ipa keeps its lexicons
+process-wide, so an enabled lexicon serves every orthography2ipa caller in the
+process, and ``packaged_lexicons=False`` does not disable one that another
+caller enabled.
 """
 from typing import Dict, List
 
@@ -29,14 +36,18 @@ class Orthography2IPAPhonemizer(BasePhonemizer):
     lazily on first use and cached for the lifetime of the instance.
     """
 
-    def __init__(self):
+    def __init__(self, packaged_lexicons: bool = True):
         super().__init__(alphabet=Alphabet.IPA)
+        self.packaged_lexicons = packaged_lexicons
         self._cache: Dict[str, object] = {}
 
     def _engine(self, resolved_lang: str):
         """Return, lazily creating and caching, the G2P engine for *resolved_lang*."""
         if resolved_lang not in self._cache:
             import orthography2ipa
+            if (self.packaged_lexicons and resolved_lang
+                    in orthography2ipa.packaged_lexicon_sources()):
+                orthography2ipa.use_packaged_lexicons([resolved_lang])
             self._cache[resolved_lang] = orthography2ipa.G2P(resolved_lang)
         return self._cache[resolved_lang]
 

{scriptconv-0.0.4a35 → scriptconv-0.0.4a37}/scriptconv/phonemizers/registry.py

@@ -106,13 +106,15 @@ def get_phonemizer(phonemizer: Phonemizer,
 
     ``model`` is forwarded to model-backed phonemizers (ByT5/Charsiu/
     DeepPhonemizer); ``alphabet`` to those with a selectable output alphabet.
-    Extra ``kwargs`` (e.g. ``normalizer=``) pass to the constructor.
+    Extra ``kwargs`` pass to the constructor, except ``normalizer=`` and
+    ``keep_punctuation=``, which are set on the instance.
     """
     phonemizer = Phonemizer(phonemizer)
     cls = get_phonemizer_class(phonemizer)
     # normalizer is a plain BasePhonemizer attribute; set it after
     # construction so wrapper __init__ signatures stay untouched
     normalizer = kwargs.pop("normalizer", None)
+    keep_punctuation = kwargs.pop("keep_punctuation", None)
     if phonemizer in (_P.BYT5, _P.CHARSIU, _P.DEEPPHONEMIZER):
         inst = cls(model, **kwargs)
     else:
@@ -131,6 +133,8 @@ def get_phonemizer(phonemizer: Phonemizer,
         inst = cls(**kwargs)
     if normalizer is not None:
         inst.normalizer = normalizer
+    if keep_punctuation is not None:
+        inst.keep_punctuation = keep_punctuation
     return inst
 
 

{scriptconv-0.0.4a35 → scriptconv-0.0.4a37}/scriptconv/version.py

@@ -2,7 +2,7 @@
 VERSION_MAJOR = 0
 VERSION_MINOR = 0
 VERSION_BUILD = 4
-VERSION_ALPHA = 35
+VERSION_ALPHA = 37
 # END_VERSION_BLOCK
 
 VERSION_STR = f"{VERSION_MAJOR}.{VERSION_MINOR}.{VERSION_BUILD}"

{scriptconv-0.0.4a35 → scriptconv-0.0.4a37/scriptconv.egg-info}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: scriptconv
-Version: 0.0.4a35
+Version: 0.0.4a37
 Summary: Zero-dependency script & phoneme-notation core — ISO-15924 detection & metadata, IPA↔ARPABET/X-SAMPA/Lexique/Kirshenbaum/Cotovía/RFE, Buckwalter↔Arabic, Hangul→jamo, kana
 License: Apache-2.0
 Project-URL: Homepage, https://github.com/TigreGotico/scriptconv
@@ -11,13 +11,12 @@ Classifier: Intended Audience :: Developers
 Classifier: Intended Audience :: Science/Research
 Classifier: Topic :: Text Processing :: Linguistic
 Classifier: Programming Language :: Python :: 3
-Classifier: Programming Language :: Python :: 3.10
 Classifier: Programming Language :: Python :: 3.11
 Classifier: Programming Language :: Python :: 3.12
 Classifier: Programming Language :: Python :: 3.13
 Classifier: License :: OSI Approved :: Apache Software License
 Classifier: Operating System :: OS Independent
-Requires-Python: >=3.10
+Requires-Python: >=3.11
 Description-Content-Type: text/markdown
 License-File: LICENSE
 Provides-Extra: ja
@@ -54,7 +53,8 @@ Requires-Dist: onnxruntime; extra == "byt5"
 Requires-Dist: numpy; extra == "byt5"
 Provides-Extra: o2i
 Requires-Dist: scriptconv[phonemizers]; extra == "o2i"
-Requires-Dist: orthography2ipa; extra == "o2i"
+Requires-Dist: orthography2ipa>=7.112.0a1; extra == "o2i"
+Requires-Dist: o2i-lexicons>=0.1.0a1; extra == "o2i"
 Provides-Extra: eu
 Requires-Dist: scriptconv[phonemizers]; extra == "eu"
 Requires-Dist: euskaphone; extra == "eu"
@@ -133,6 +133,9 @@ Requires-Dist: bifonia>=0.1.1; extra == "test"
 Requires-Dist: stressonnx>=0.0.3a2; extra == "test"
 Requires-Dist: tugaphone>=1.2.0a1; extra == "test"
 Requires-Dist: ghana-g2p>=0.1.1; extra == "test"
+Requires-Dist: espyak; extra == "test"
+Requires-Dist: orthography2ipa>=7.112.0a1; extra == "test"
+Requires-Dist: o2i-lexicons>=0.1.0a1; extra == "test"
 Requires-Dist: tomli; python_version < "3.11" and extra == "test"
 Dynamic: license-file
 
@@ -390,7 +393,7 @@ g.convert("bom dia", "text", "ipa", lang="pt")     # 'ˈbõ ˈdʒiɐ'
 g.can_convert("text", "arpa")                      # True (chains through IPA)
 ```
 
-Two design points worth knowing:
+Three design points worth knowing:
 
 - **Normalization is injectable.** TTS stacks expand numbers and dates
   before phonemizing, and that needs language resources scriptconv does
@@ -405,6 +408,17 @@ Two design points worth knowing:
   overrides it with a path or callable; the cache location is set via
   `SCRIPTCONV_CACHE` or `XDG_CACHE_HOME`).
 
+- **The orthography2ipa phonemizer uses the packaged lexicons.** The `o2i`
+  extra installs [o2i-lexicons](https://github.com/TigreGotico/o2i-lexicons),
+  which holds the words that the orthography2ipa rules get wrong, such as
+  en-GB ⟨yacht⟩. The key `packaged_lexicons` (default `True`) enables the
+  packaged lexicon for each language that the phonemizer transcribes, when a
+  package serves that language. Set it to `False` to use the rules only:
+  `get_phonemizer(Phonemizer.ORTHOGRAPHY2IPA, packaged_lexicons=False)`.
+  orthography2ipa keeps its lexicons for the whole process. After one caller
+  enables a lexicon, every orthography2ipa caller in the process uses it, and
+  `packaged_lexicons=False` does not disable it again.
+
 **Pre-G2P disambiguation.** `add_diacritics(text, lang, model=None)`
 restores information ordinary orthography omits but a G2P needs, before
 phonemization. It covers Hebrew niqqud (phonikud), Arabic tashkeel

{scriptconv-0.0.4a35 → scriptconv-0.0.4a37}/scriptconv.egg-info/SOURCES.txt

@@ -515,9 +515,11 @@ tests/test_phonemizers_friendly_import_errors.py
 tests/test_phonemizers_ghana.py
 tests/test_phonemizers_halabi_iqra.py
 tests/test_phonemizers_matoub_kab.py
+tests/test_phonemizers_o2i_lexicons.py
 tests/test_phonemizers_phonetisaurus.py
 tests/test_phonemizers_phonetisaurus_engine.py
 tests/test_phonemizers_pt_tugaphone.py
+tests/test_phonemizers_punctuation.py
 tests/test_phonemizers_ru.py
 tests/test_readings.py
 tests/test_readings_zh.py

{scriptconv-0.0.4a35 → scriptconv-0.0.4a37}/scriptconv.egg-info/requires.txt

@@ -78,7 +78,8 @@ mwl_phonemizer
 
 [o2i]
 scriptconv[phonemizers]
-orthography2ipa
+orthography2ipa>=7.112.0a1
+o2i-lexicons>=0.1.0a1
 
 [phonemizers]
 quebra-frases
@@ -122,6 +123,9 @@ bifonia>=0.1.1
 stressonnx>=0.0.3a2
 tugaphone>=1.2.0a1
 ghana-g2p>=0.1.1
+espyak
+orthography2ipa>=7.112.0a1
+o2i-lexicons>=0.1.0a1
 
 [test:python_version < "3.11"]
 tomli

scriptconv-0.0.4a37/tests/test_phonemizers_o2i_lexicons.py

@@ -0,0 +1,68 @@
+"""The orthography2ipa phonemizer enables the packaged o2i-lexicons by default.
+
+The expected IPA is read from the o2i-lexicons data file, not from engine
+output. orthography2ipa keeps lexicons process-wide, so every test starts and
+ends with ``clear_lexicons()``.
+"""
+import unittest
+
+import orthography2ipa
+from o2i_lexicons import DATA_DIR
+
+from scriptconv.phonemizers.o2ipa import Orthography2IPAPhonemizer
+
+STRESS = str.maketrans("", "", "ˈˌ")
+
+
+def lexicon_ipa(code: str, word: str) -> str:
+    """The IPA that the packaged TSV file for *code* lists for *word*."""
+    for path in sorted((DATA_DIR / code).glob("*.tsv")):
+        for line in path.read_text(encoding="utf-8").splitlines():
+            entry, _, ipa = line.partition("\t")
+            if entry == word:
+                return ipa
+    raise LookupError(f"{word!r} is not in the packaged {code} lexicon")
+
+
+class TestPackagedLexicons(unittest.TestCase):
+    def setUp(self):
+        orthography2ipa.clear_lexicons()
+
+    def tearDown(self):
+        orthography2ipa.clear_lexicons()
+
+    def test_default_uses_the_packaged_lexicon(self):
+        expected = lexicon_ipa("en-GB", "yacht")
+        out = Orthography2IPAPhonemizer().phonemize_string("yacht", "en-GB")
+        self.assertEqual(out.translate(STRESS), expected)
+
+    def test_bare_english_tag_uses_the_packaged_lexicon(self):
+        expected = lexicon_ipa("en-GB", "yacht")
+        out = Orthography2IPAPhonemizer().phonemize_string("yacht", "en")
+        self.assertEqual(out.translate(STRESS), expected)
+
+    def test_off_equals_plain_orthography2ipa(self):
+        out = Orthography2IPAPhonemizer(packaged_lexicons=False) \
+            .phonemize_string("yacht", "en-GB")
+        self.assertEqual(out, orthography2ipa.G2P("en-GB").transcribe("yacht"))
+        self.assertNotEqual(out.translate(STRESS), lexicon_ipa("en-GB", "yacht"))
+        self.assertEqual(orthography2ipa.available_lexicon_codes(), [])
+
+    def test_enables_only_the_languages_it_transcribes(self):
+        Orthography2IPAPhonemizer().phonemize_string("yacht", "en-GB")
+        self.assertEqual(orthography2ipa.available_lexicon_codes(), ["en-GB"])
+
+    def test_language_without_a_packaged_lexicon_uses_the_rules(self):
+        out = Orthography2IPAPhonemizer().phonemize_string("Kaixo", "eu")
+        self.assertEqual(out, orthography2ipa.G2P("eu").transcribe("Kaixo"))
+        self.assertEqual(orthography2ipa.available_lexicon_codes(), [])
+
+    def test_registry_forwards_the_key(self):
+        from scriptconv.phonemizers import Phonemizer, get_phonemizer
+        off = get_phonemizer(Phonemizer.ORTHOGRAPHY2IPA, packaged_lexicons=False)
+        self.assertFalse(off.packaged_lexicons)
+        self.assertTrue(get_phonemizer(Phonemizer.ORTHOGRAPHY2IPA).packaged_lexicons)
+
+
+if __name__ == "__main__":
+    unittest.main()