scriptconv 0.0.4a34__tar.gz → 0.0.4a36__tar.gz

{scriptconv-0.0.4a34/scriptconv.egg-info → scriptconv-0.0.4a36}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: scriptconv
-Version: 0.0.4a34
+Version: 0.0.4a36
 Summary: Zero-dependency script & phoneme-notation core — ISO-15924 detection & metadata, IPA↔ARPABET/X-SAMPA/Lexique/Kirshenbaum/Cotovía/RFE, Buckwalter↔Arabic, Hangul→jamo, kana
 License: Apache-2.0
 Project-URL: Homepage, https://github.com/TigreGotico/scriptconv
@@ -133,6 +133,7 @@ Requires-Dist: bifonia>=0.1.1; extra == "test"
 Requires-Dist: stressonnx>=0.0.3a2; extra == "test"
 Requires-Dist: tugaphone>=1.2.0a1; extra == "test"
 Requires-Dist: ghana-g2p>=0.1.1; extra == "test"
+Requires-Dist: espyak; extra == "test"
 Requires-Dist: tomli; python_version < "3.11" and extra == "test"
 Dynamic: license-file
 

{scriptconv-0.0.4a34 → scriptconv-0.0.4a36}/pyproject.toml

@@ -63,7 +63,7 @@ zh-phonemizers = ["scriptconv[phonemizers]", "jieba", "pypinyin>=0.50,<1", "xpin
 en-phonemizers = ["scriptconv[phonemizers]", "g2p_en", "openphonemizer", "deep-phonemizer", "requests"]
 ar-phonemizers = ["scriptconv[phonemizers]", "arbtok", "ovos-number-parser"]
 shami = ["scriptconv[phonemizers]", "regex"]
-test = ["pytest", "pytest-timeout", "pykakasi>=2.3,<3", "pypinyin>=0.50,<1", "quebra-frases", "langcodes", "regex", "ovos-number-parser", "bifonia>=0.1.1", "stressonnx>=0.0.3a2", "tugaphone>=1.2.0a1", "ghana-g2p>=0.1.1", "tomli; python_version < '3.11'"]
+test = ["pytest", "pytest-timeout", "pykakasi>=2.3,<3", "pypinyin>=0.50,<1", "quebra-frases", "langcodes", "regex", "ovos-number-parser", "bifonia>=0.1.1", "stressonnx>=0.0.3a2", "tugaphone>=1.2.0a1", "ghana-g2p>=0.1.1", "espyak", "tomli; python_version < '3.11'"]
 
 [project.urls]
 Homepage = "https://github.com/TigreGotico/scriptconv"

{scriptconv-0.0.4a34 → scriptconv-0.0.4a36}/scriptconv/phonemizers/africa.py

@@ -90,18 +90,41 @@ class AfricaG2PPhonemizer(BasePhonemizer):
     @classmethod
     def get_lang(cls, target_lang: str) -> str:
         """
-        Resolve *target_lang* to a supported africa-g2p ISO 639-3 code.
-
-        Only the primary subtag is matched (e.g. ``tw-GH`` -> ``twi`` is *not*
-        attempted — africa-g2p keys its rule files by exact ISO 639-3 code, not
-        BCP-47 macrolanguage/region mapping, so this is an exact-match lookup,
-        not the closest-match fuzzing :meth:`BasePhonemizer.match_lang` does).
+        Resolve *target_lang* to a supported africa-g2p rule-file name.
+
+        Two exact lookups, in order, and no fuzzing (africa-g2p keys its rule
+        files by name, not by BCP-47 macrolanguage/region mapping, so
+        :meth:`BasePhonemizer.match_lang`'s closest-match search does not
+        apply):
+
+        1. the whole tag, lowercased, first as written and then with ``_``
+           read as ``-``. 11 of the rule
+           files carry a region suffix rather than a bare ISO 639-3 code
+           (``hau-nigeria``, ``hau-niger``, ``dop-benin``, ``dop-benin2``,
+           ``ngb-zaire``, ``ngb-zaire2``, ``sag-congo``,
+           ``sag-central_african_republic``, ``sef-cote_d_ivoire``,
+           ``sef-cote_d_ivoire2``, ``snk-senegal``), and before this step none
+           of them could be reached at all: the primary subtag ``hau`` is not
+           a rule file, so Hausa raised.
+        2. the primary subtag (``twi-GH`` -> ``twi``).
+
+        A bare code whose only rule files carry region suffixes still raises.
+        Picking one region for ``hau`` would be a claim about which variety a
+        caller meant, and this wrapper does not make it.
 
         Raises:
             ValueError: If africa-g2p has no rule file for *target_lang*.
         """
+        langs = cls.supported_langs()
+        lowered = target_lang.lower()
+        # Two of the names keep an underscore of their own
+        # (sef-cote_d_ivoire, sag-central_african_republic), so the tag is
+        # tried as written before ``_`` is read as ``-``.
+        for whole in (lowered, lowered.replace("_", "-")):
+            if whole in langs:
+                return whole
         key = _primary_subtag(target_lang)
-        if key in cls.supported_langs():
+        if key in langs:
             return key
         raise ValueError(f"africa-g2p: unsupported language {target_lang!r}")
 

{scriptconv-0.0.4a34 → scriptconv-0.0.4a36}/scriptconv/phonemizers/base.py

@@ -34,6 +34,52 @@ RawPhonemizedChunks = List[Tuple[str, str, bool]]
 
 PhonemizedChunks = list[list[str]]
 
+# Clause punctuation kept by ``keep_punctuation``, mapped to the symbol a voice
+# is trained on. These are the six clause types Piper appends after every
+# espeak-ng clause (piper-phonemize ``src/phonemize.cpp``): a full stop,
+# question mark and exclamation mark stand alone, a comma, colon and semicolon
+# are followed by a space. Arabic script has its own comma, semicolon and
+# question mark; they map onto the same symbols.
+CLAUSE_PUNCTUATION = {".": ".", "?": "?", "!": "!", ",": ",", ":": ":", ";": ";",
+                      "\u060c": ",", "\u061b": ";", "\u061f": "?"}
+_PAUSE_PUNCTUATION = {",", ":", ";"}
+# A clause ends at punctuation followed by whitespace, as in espeak-ng, so the
+# "," in "10,4" or the ":" in "16:30" never splits. A lone "." is left to the
+# sentence tokenizer, which knows abbreviations; inside a sentence an ellipsis
+# or "|" still ends a clause, as in chunk_text, but carries no symbol (espeak-ng
+# reads an ellipsis as a spoken-name pause, which Piper does not keep).
+_CLAUSE_BREAK = re.compile(r"(?:\.\.+|\u2026|[,:;!?|\u060c\u061b\u061f])[^\w\s]*(?=\s)")
+_TRAILING_PUNCTUATION = re.compile(r"[^\w\s]+\s*$")
+_LANG_FLAGS = re.compile(r"\([^)]+\)")
+
+
+def _clause_mark(punctuation: str) -> Optional[str]:
+    """The symbol a run of clause-final punctuation is kept as, if any."""
+    if ".." in punctuation or "\u2026" in punctuation:
+        return None
+    for char in punctuation:
+        if char in CLAUSE_PUNCTUATION:
+            return CLAUSE_PUNCTUATION[char]
+    return None
+
+
+def split_clauses(sentence: str) -> List[Tuple[str, Optional[str]]]:
+    """Split one sentence into ``(clause, mark)`` pairs.
+
+    ``mark`` is the symbol from :data:`CLAUSE_PUNCTUATION` that ends the
+    clause, or ``None`` when the clause ends without one (no punctuation, an
+    ellipsis, ``|``). The clause text still carries its own punctuation.
+    """
+    clauses: List[Tuple[str, Optional[str]]] = []
+    start = 0
+    for match in _CLAUSE_BREAK.finditer(sentence):
+        clauses.append((sentence[start:match.start()], _clause_mark(match.group())))
+        start = match.end()
+    tail = sentence[start:]
+    trailing = _TRAILING_PUNCTUATION.search(tail)
+    clauses.append((tail, _clause_mark(trailing.group()) if trailing else None))
+    return clauses
+
 
 class MissingLanguageError(ValueError):
     """No language reached :meth:`BasePhonemizer.match_lang` at all.
@@ -75,6 +121,18 @@ def _check_alphabet(phonemizer, alphabet: Alphabet,
 
 
 class BasePhonemizer(metaclass=abc.ABCMeta):
+    keep_punctuation: bool = False
+    """Keep clause punctuation (``, . ? ! : ;``) in the phoneme output.
+
+    Off by default, so voices trained without punctuation receive exactly the
+    sequences they were trained on. When on, :meth:`phonemize_to_list` and
+    :meth:`phonemize_lazy` phonemize clause by clause and append each clause's
+    symbol the way Piper does, and the two return the same phonemes for the
+    same text. A backend that overrides either method owns its own output and
+    ignores this flag, since honouring it on one side only would make training
+    and synthesis disagree.
+    """
+
     def __init__(self, alphabet: Alphabet = Alphabet.UNICODE,
                  normalizer: Optional[Callable[[str, str], str]] = None):
         super().__init__()
@@ -88,7 +146,17 @@ class BasePhonemizer(metaclass=abc.ABCMeta):
         raise NotImplementedError
 
     def phonemize_to_list(self, text: str, lang: str) -> List[str]:
-        return list(self.phonemize_string(text, lang))
+        """Phonemes of *text* as one flat list, without normalization.
+
+        Clause breaks that a backend prints as ``"\\n"`` (espeak-ng does at
+        every comma, question mark and exclamation mark) are not phonemes and
+        are dropped; with :attr:`keep_punctuation` the clause's punctuation
+        takes their place.
+        """
+        if self._keeps_punctuation():
+            return [p for sentence in self._punctuated_sentences(text, lang)
+                    for p in sentence]
+        return [p for p in self.phonemize_string(text, lang) if p != "\n"]
 
     def phonemize(self, text: str, lang: str) -> PhonemizedChunks:
         # PhonemizedChunks is list[list[str]]; empty text yields no
@@ -114,6 +182,9 @@ class BasePhonemizer(metaclass=abc.ABCMeta):
             return
         if self.normalizer is not None:
             text = self.normalizer(text, lang)
+        if self._keeps_punctuation():
+            yield from self._punctuated_sentences(text, lang)
+            return
         for chunk, punct, eos in self.chunk_text(text):
             phoneme_str = self.phonemize_string(self.remove_punctuation(chunk), lang)
             # Filter out (lang) switch (flags) that surround words from languages
@@ -123,6 +194,44 @@ class BasePhonemizer(metaclass=abc.ABCMeta):
             # its own sentence.
             yield list(phoneme_str)
 
+    def _keeps_punctuation(self) -> bool:
+        cls = type(self)
+        return (self.keep_punctuation
+                and cls.phonemize_to_list is BasePhonemizer.phonemize_to_list
+                and cls.phonemize_lazy is BasePhonemizer.phonemize_lazy)
+
+    def _punctuated_sentences(self, text: str, lang: str) -> Iterator[List[str]]:
+        """Per-sentence phonemes with each clause's punctuation appended.
+
+        Each clause is phonemized on its own with its punctuation removed, so
+        the backend sees the same words in training and in synthesis, and the
+        clause's symbol is appended after it: bare for ``. ? !``, followed by a
+        space for ``, : ;``.
+        """
+        try:
+            from quebra_frases import sentence_tokenize
+        except ImportError:
+            raise ImportError(
+                "sentence chunking needs quebra-frases — install with "
+                "`pip install scriptconv[phonemizers]`") from None
+        for sentence in sentence_tokenize(text):
+            phonemes: List[str] = []
+            for clause, mark in split_clauses(sentence):
+                clause = self.remove_punctuation(clause)
+                if not clause:
+                    continue
+                phoneme_str = _LANG_FLAGS.sub("", self.phonemize_string(clause, lang))
+                clause_phonemes = [p for p in phoneme_str if p != "\n"]
+                if not clause_phonemes:
+                    continue
+                phonemes.extend(clause_phonemes)
+                if mark:
+                    phonemes.append(mark)
+                    if mark in _PAUSE_PUNCTUATION:
+                        phonemes.append(" ")
+            if phonemes:
+                yield phonemes
+
     @staticmethod
     def _process_phones(raw_phones: RawPhonemizedChunks) -> PhonemizedChunks:
         """Text to phonemes grouped by sentence."""

{scriptconv-0.0.4a34 → scriptconv-0.0.4a36}/scriptconv/phonemizers/registry.py

@@ -106,13 +106,15 @@ def get_phonemizer(phonemizer: Phonemizer,
 
     ``model`` is forwarded to model-backed phonemizers (ByT5/Charsiu/
     DeepPhonemizer); ``alphabet`` to those with a selectable output alphabet.
-    Extra ``kwargs`` (e.g. ``normalizer=``) pass to the constructor.
+    Extra ``kwargs`` pass to the constructor, except ``normalizer=`` and
+    ``keep_punctuation=``, which are set on the instance.
     """
     phonemizer = Phonemizer(phonemizer)
     cls = get_phonemizer_class(phonemizer)
     # normalizer is a plain BasePhonemizer attribute; set it after
     # construction so wrapper __init__ signatures stay untouched
     normalizer = kwargs.pop("normalizer", None)
+    keep_punctuation = kwargs.pop("keep_punctuation", None)
     if phonemizer in (_P.BYT5, _P.CHARSIU, _P.DEEPPHONEMIZER):
         inst = cls(model, **kwargs)
     else:
@@ -131,6 +133,8 @@ def get_phonemizer(phonemizer: Phonemizer,
         inst = cls(**kwargs)
     if normalizer is not None:
         inst.normalizer = normalizer
+    if keep_punctuation is not None:
+        inst.keep_punctuation = keep_punctuation
     return inst
 
 

{scriptconv-0.0.4a34 → scriptconv-0.0.4a36}/scriptconv/version.py

@@ -2,7 +2,7 @@
 VERSION_MAJOR = 0
 VERSION_MINOR = 0
 VERSION_BUILD = 4
-VERSION_ALPHA = 34
+VERSION_ALPHA = 36
 # END_VERSION_BLOCK
 
 VERSION_STR = f"{VERSION_MAJOR}.{VERSION_MINOR}.{VERSION_BUILD}"

{scriptconv-0.0.4a34 → scriptconv-0.0.4a36/scriptconv.egg-info}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: scriptconv
-Version: 0.0.4a34
+Version: 0.0.4a36
 Summary: Zero-dependency script & phoneme-notation core — ISO-15924 detection & metadata, IPA↔ARPABET/X-SAMPA/Lexique/Kirshenbaum/Cotovía/RFE, Buckwalter↔Arabic, Hangul→jamo, kana
 License: Apache-2.0
 Project-URL: Homepage, https://github.com/TigreGotico/scriptconv
@@ -133,6 +133,7 @@ Requires-Dist: bifonia>=0.1.1; extra == "test"
 Requires-Dist: stressonnx>=0.0.3a2; extra == "test"
 Requires-Dist: tugaphone>=1.2.0a1; extra == "test"
 Requires-Dist: ghana-g2p>=0.1.1; extra == "test"
+Requires-Dist: espyak; extra == "test"
 Requires-Dist: tomli; python_version < "3.11" and extra == "test"
 Dynamic: license-file
 

{scriptconv-0.0.4a34 → scriptconv-0.0.4a36}/scriptconv.egg-info/SOURCES.txt

@@ -518,6 +518,7 @@ tests/test_phonemizers_matoub_kab.py
 tests/test_phonemizers_phonetisaurus.py
 tests/test_phonemizers_phonetisaurus_engine.py
 tests/test_phonemizers_pt_tugaphone.py
+tests/test_phonemizers_punctuation.py
 tests/test_phonemizers_ru.py
 tests/test_readings.py
 tests/test_readings_zh.py

{scriptconv-0.0.4a34 → scriptconv-0.0.4a36}/scriptconv.egg-info/requires.txt

@@ -122,6 +122,7 @@ bifonia>=0.1.1
 stressonnx>=0.0.3a2
 tugaphone>=1.2.0a1
 ghana-g2p>=0.1.1
+espyak
 
 [test:python_version < "3.11"]
 tomli

{scriptconv-0.0.4a34 → scriptconv-0.0.4a36}/tests/test_phonemizers_africa.py

@@ -47,6 +47,30 @@ class TestAfricaG2PPhonemizer(unittest.TestCase):
         self.assertIn("twi", AfricaG2PPhonemizer.supported_langs())
         self.assertGreater(len(AfricaG2PPhonemizer.supported_langs()), 300)
 
+    def test_region_suffixed_rule_files_resolve_by_their_own_name(self):
+        # 11 rule files carry a region suffix instead of a bare ISO 639-3
+        # code; each must be selectable by that name (T-2701).
+        for code in ("hau-nigeria", "hau-niger", "dop-benin", "ngb-zaire",
+                     "sag-congo", "sef-cote_d_ivoire", "snk-senegal"):
+            self.assertEqual(AfricaG2PPhonemizer.get_lang(code), code)
+            self.assertIn(code, AfricaG2PPhonemizer.supported_langs())
+
+    def test_a_region_suffixed_code_phonemizes(self):
+        p = AfricaG2PPhonemizer(alphabet=Alphabet.IPA)
+        self.assertEqual(p.phonemize_string("ƙasa", "hau-nigeria"),
+                         "kʼ a s a")
+
+    def test_an_underscore_is_read_as_a_hyphen(self):
+        self.assertEqual(AfricaG2PPhonemizer.get_lang("HAU_NIGERIA"),
+                         "hau-nigeria")
+
+    def test_a_bare_code_with_only_region_files_still_raises(self):
+        # africa-g2p ships no bare 'hau'. Choosing a region for the caller
+        # would be a claim this wrapper does not make.
+        self.assertNotIn("hau", AfricaG2PPhonemizer.supported_langs())
+        with self.assertRaises(ValueError):
+            AfricaG2PPhonemizer.get_lang("hau")
+
     def test_get_lang_resolves_exact_iso_639_3_code(self):
         self.assertEqual(AfricaG2PPhonemizer.get_lang("twi"), "twi")
         # region subtag is stripped to the primary subtag before lookup

scriptconv-0.0.4a36/tests/test_phonemizers_punctuation.py

@@ -0,0 +1,175 @@
+"""Clause punctuation in phonemizer output.
+
+By default no punctuation reaches the phonemes: voices trained that way must
+keep receiving the sequences they learned. ``keep_punctuation`` opts in to the
+Piper convention, where every clause is followed by its symbol (``. ? !`` bare,
+``, : ;`` followed by a space). Clause breaks that espeak-ng prints as ``"\\n"``
+are never returned as phonemes, in either mode.
+
+The espeak tests run against the espeak-ng binary when it is installed and
+against the pure-Python espyak port otherwise, so they assert the punctuation
+and its position, not the exact phonemes the two produce for the words.
+"""
+import re
+
+import pytest
+
+from scriptconv.phonemizers import get_phonemizer
+from scriptconv.phonemizers.base import BasePhonemizer
+from scriptconv.phonemizers.enums import Phonemizer
+from scriptconv.phonemizers.mul import EspeakPhonemizer
+
+SONGS = "Songs near this quiet river every morning, number three."
+
+MULTILINGUAL = {
+    "en-us": ["Songs near this quiet river every morning, number three.",
+              "Is it ready? Yes! We leave at 16:30; bring 10,4 kg... Mr. Smith said: \"fine\"."],
+    "pt": ["Olá, como estás? Eu estou bem! Vamos às 16:30; leva 10,4 kg."],
+    "es": ["¿Dónde está la biblioteca? ¡Allí, al lado del parque!"],
+    "de": ["Guten Morgen, wie geht es dir? Mir geht es gut!"],
+    "ar": ["مرحبا، كيف حالك؟ أنا بخير!"],
+}
+
+
+class CliLike(BasePhonemizer):
+    """Behaves like the espeak-ng CLI: drops punctuation, prints a newline at
+    every clause break, and nothing for a full stop."""
+
+    def phonemize_string(self, text, lang):
+        out = []
+        for char in text:
+            if char in ",?!;:":
+                out.append("\n")
+            elif char not in ".":
+                out.append(char.upper())
+        return "".join(out).strip()
+
+
+@pytest.fixture
+def espeak():
+    return EspeakPhonemizer()
+
+
+def flat(sentences):
+    return [p for sentence in sentences for p in sentence]
+
+
+class TestDefaultKeepsNoPunctuation:
+    def test_flag_is_off_by_default(self, espeak):
+        assert espeak.keep_punctuation is False
+        assert get_phonemizer(Phonemizer.ESPEAK).keep_punctuation is False
+
+    def test_no_punctuation_in_espeak_output(self, espeak):
+        for lang, texts in MULTILINGUAL.items():
+            for text in texts:
+                assert not set(",.?!:;") & set(espeak.phonemize_to_list(text, lang))
+                assert not set(",.?!:;") & set(flat(espeak.phonemize(text, lang)))
+
+    def test_clause_breaks_are_not_phonemes(self):
+        assert CliLike().phonemize_string("one, two? three", "en") == "ONE\n TWO\n THREE"
+        assert CliLike().phonemize_to_list("one, two? three", "en") == list("ONE TWO THREE")
+
+    def test_no_newline_from_espeak(self, espeak):
+        for lang, texts in MULTILINGUAL.items():
+            for text in texts:
+                assert "\n" not in espeak.phonemize_to_list(text, lang)
+
+    def test_default_matches_the_previous_definitions(self, espeak):
+        # phonemize_to_list was list(phonemize_string) and phonemize_lazy
+        # phonemized each chunk_text chunk with its punctuation removed; the
+        # default output is those, less the newline clause breaks
+        for lang, texts in MULTILINGUAL.items():
+            for text in texts:
+                raw = espeak.phonemize_string(text, lang)
+                assert espeak.phonemize_to_list(text, lang) == [p for p in raw if p != "\n"]
+                previous = [
+                    list(re.sub(r"\([^)]+\)", "", espeak.phonemize_string(
+                        espeak.remove_punctuation(chunk), lang)))
+                    for chunk, _, _ in espeak.chunk_text(text)]
+                assert espeak.phonemize(text, lang) == previous
+                assert list(espeak.phonemize_lazy(text, lang)) == previous
+
+    def test_synthesis_path_is_unchanged(self):
+        # chunked, punctuation removed before the backend, no symbol appended
+        assert CliLike().phonemize("one, two. three!", "en") == [
+            list("ONE TWO"), list("THREE")]
+
+
+class TestKeepPunctuation:
+    def test_espeak_keeps_comma_and_full_stop(self, espeak):
+        espeak.keep_punctuation = True
+        phonemes = espeak.phonemize_to_list(SONGS, "en-us")
+        text = "".join(phonemes)
+        assert phonemes.count(",") == 1 and phonemes.count(".") == 1
+        assert phonemes[-1] == "."
+        assert text.index(", ") < text.index("nˈʌmb")  # before "number"
+        assert "\n" not in phonemes
+
+    def test_factory_sets_the_flag(self):
+        phonemizer = get_phonemizer(Phonemizer.ESPEAK, keep_punctuation=True)
+        assert phonemizer.keep_punctuation is True
+        assert "," in phonemizer.phonemize_to_list(SONGS, "en-us")
+
+    def test_training_and_synthesis_agree(self, espeak):
+        espeak.keep_punctuation = True
+        for lang, texts in MULTILINGUAL.items():
+            for text in texts:
+                train = espeak.phonemize_to_list(text, lang)
+                synth = flat(espeak.phonemize(text, lang))
+                assert train == synth, (lang, text)
+                assert flat(espeak.phonemize_lazy(text, lang)) == synth
+                assert "\n" not in train
+
+    def test_symbols_follow_piper(self):
+        phonemizer = CliLike()
+        phonemizer.keep_punctuation = True
+        assert phonemizer.phonemize_to_list("one, two: three; four? five! six.", "en") == (
+            list("ONE") + [",", " "] + list("TWO") + [":", " "] + list("THREE")
+            + [";", " "] + list("FOUR") + ["?"] + list("FIVE") + ["!"] + list("SIX") + ["."])
+
+    def test_sentences_stay_separate_for_synthesis(self):
+        phonemizer = CliLike()
+        phonemizer.keep_punctuation = True
+        assert phonemizer.phonemize("One, two. Three?", "en") == [
+            list("ONE") + [",", " "] + list("TWO") + ["."], list("THREE") + ["?"]]
+
+    def test_arabic_marks_map_to_ascii(self, espeak):
+        espeak.keep_punctuation = True
+        phonemes = espeak.phonemize_to_list("مرحبا، كيف حالك؟ أنا بخير!", "ar")
+        assert [p for p in phonemes if p in ",?!"] == [",", "?", "!"]
+
+    def test_normalizer_runs_only_on_synthesis(self):
+        phonemizer = CliLike()
+        phonemizer.keep_punctuation = True
+        phonemizer.normalizer = lambda text, lang: text.replace("3", "three")
+        train = phonemizer.phonemize_to_list("number three.", "en")
+        assert train[-1] == "."
+        assert flat(phonemizer.phonemize("number 3.", "en")) == train
+
+    def test_a_backend_with_its_own_list_ignores_the_flag(self):
+        class OwnList(CliLike):
+            def phonemize_to_list(self, text, lang):
+                return list(self.phonemize_string(self.remove_punctuation(text), lang))
+
+        phonemizer = OwnList()
+        phonemizer.keep_punctuation = True
+        # one side overridden: honouring the flag on the other side alone would
+        # make training and synthesis disagree
+        assert phonemizer.phonemize("one, two.", "en") == [list("ONE TWO")]
+
+
+class TestSplitClauses:
+    @pytest.mark.parametrize("sentence, clauses", [
+        ("one, two.", [("one", ","), (" two.", ".")]),
+        ("at 16:30, ten", [("at 16:30", ","), (" ten", None)]),
+        ("10,4 kg", [("10,4 kg", None)]),
+        ("wait... go", [("wait", None), (" go", None)]),
+        ("wait…", [("wait…", None)]),
+        ('he said "hi," then left.', [('he said "hi', ","), (" then left.", ".")]),
+        ("really?! yes", [("really", "?"), (" yes", None)]),
+        ("Mr. Smith", [("Mr. Smith", None)]),
+        ("مرحبا، كيف", [("مرحبا", ","), (" كيف", None)]),
+    ])
+    def test_split(self, sentence, clauses):
+        from scriptconv.phonemizers.base import split_clauses
+        assert split_clauses(sentence) == clauses