scriptconv 0.0.3a10__tar.gz → 0.0.3a12__tar.gz

{scriptconv-0.0.3a10 → scriptconv-0.0.3a12}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: scriptconv
-Version: 0.0.3a10
+Version: 0.0.3a12
 Summary: Zero-dependency script & phoneme-notation core — ISO-15924 detection & metadata, IPA↔ARPABET/X-SAMPA/Lexique/Kirshenbaum/Cotovía/RFE, Buckwalter↔Arabic, Hangul→jamo, kana
 License: Apache-2.0
 Project-URL: Homepage, https://github.com/TigreGotico/scriptconv
@@ -24,11 +24,67 @@ Provides-Extra: ja
 Requires-Dist: pykakasi<3,>=2.3; extra == "ja"
 Provides-Extra: zh
 Requires-Dist: pypinyin<1,>=0.50; extra == "zh"
+Provides-Extra: phonemizers
+Requires-Dist: quebra-frases; extra == "phonemizers"
+Requires-Dist: langcodes; extra == "phonemizers"
+Provides-Extra: espeak
+Requires-Dist: scriptconv[phonemizers]; extra == "espeak"
+Requires-Dist: espyak; extra == "espeak"
+Provides-Extra: gruut
+Requires-Dist: scriptconv[phonemizers]; extra == "gruut"
+Requires-Dist: gruut; extra == "gruut"
+Provides-Extra: goruut
+Requires-Dist: scriptconv[phonemizers]; extra == "goruut"
+Requires-Dist: pygoruut; extra == "goruut"
+Provides-Extra: epitran
+Requires-Dist: scriptconv[phonemizers]; extra == "epitran"
+Requires-Dist: epitran; extra == "epitran"
+Provides-Extra: transphone
+Requires-Dist: scriptconv[phonemizers]; extra == "transphone"
+Requires-Dist: transphone; extra == "transphone"
+Provides-Extra: misaki
+Requires-Dist: scriptconv[phonemizers]; extra == "misaki"
+Requires-Dist: misaki; extra == "misaki"
+Provides-Extra: byt5
+Requires-Dist: scriptconv[phonemizers]; extra == "byt5"
+Requires-Dist: onnxruntime; extra == "byt5"
+Requires-Dist: numpy; extra == "byt5"
+Provides-Extra: o2i
+Requires-Dist: scriptconv[phonemizers]; extra == "o2i"
+Requires-Dist: orthography2ipa; extra == "o2i"
+Provides-Extra: eu
+Requires-Dist: scriptconv[phonemizers]; extra == "eu"
+Requires-Dist: euskaphone; extra == "eu"
+Requires-Dist: ahotts_g2p; extra == "eu"
+Provides-Extra: pt-phonemizers
+Requires-Dist: scriptconv[phonemizers]; extra == "pt-phonemizers"
+Requires-Dist: tugaphone; extra == "pt-phonemizers"
+Requires-Dist: g2p_barranquenho; extra == "pt-phonemizers"
+Provides-Extra: mwl
+Requires-Dist: scriptconv[phonemizers]; extra == "mwl"
+Requires-Dist: mwl_phonemizer; extra == "mwl"
+Provides-Extra: gl
+Requires-Dist: scriptconv[phonemizers]; extra == "gl"
+Requires-Dist: pycotovia; extra == "gl"
+Provides-Extra: he
+Requires-Dist: scriptconv[phonemizers]; extra == "he"
+Requires-Dist: phonikud-onnx; extra == "he"
+Provides-Extra: fa
+Requires-Dist: scriptconv[phonemizers]; extra == "fa"
+Requires-Dist: persian_phonemizer; extra == "fa"
+Provides-Extra: vi
+Requires-Dist: scriptconv[phonemizers]; extra == "vi"
+Requires-Dist: viphoneme; extra == "vi"
+Provides-Extra: tashkeel
+Requires-Dist: scriptconv[phonemizers]; extra == "tashkeel"
+Requires-Dist: text2tashkeel; extra == "tashkeel"
 Provides-Extra: test
 Requires-Dist: pytest; extra == "test"
 Requires-Dist: pytest-timeout; extra == "test"
 Requires-Dist: pykakasi<3,>=2.3; extra == "test"
 Requires-Dist: pypinyin<1,>=0.50; extra == "test"
+Requires-Dist: quebra-frases; extra == "test"
+Requires-Dist: langcodes; extra == "test"
 Dynamic: license-file
 
 # scriptconv
@@ -210,7 +266,7 @@ below describes the per-converter default.
 
 | Notation | `to_ipa` → `from_ipa` round-trip | `from_ipa` → `to_ipa` round-trip | Unknown-token behaviour |
 |----------|----------------------------------|----------------------------------|-------------------------|
-| **ARPABET** | Exact for base symbols; **stress digits are dropped** and `AH0`↔`AX` (schwa) is not distinguished | **Lossy** — ARPABET is an English-only inventory, so any IPA symbol outside it becomes the *unknown* placeholder | `arpa_to_ipa`: passed through unchanged. `ipa_to_arpa`: diacritics and suprasegmentals (combining marks, length/stress modifiers) are dropped; other out-of-inventory symbols are replaced with `?` by default (`unknown=` param; `unknown=""` drops) |
+| **ARPABET** | **Lossless with `stress=True`** (digits ↔ IPA `ˈ`/`ˌ` before the stressed vowel; residues: extended-ARPABET `AX` normalises to CMUdict's `AH0` spelling, and `AH0 R` fuses to the r-colored `AXR0` — stable from the IPA side). Default `stress=False` drops digits and merges `AH0`↔`AX` | **Lossy** — ARPABET is an English-only inventory, so any IPA symbol outside it becomes the *unknown* placeholder | `arpa_to_ipa`: passed through unchanged. `ipa_to_arpa`: diacritics and suprasegmentals are dropped; other out-of-inventory symbols follow `errors=` (default `"replace"` → `?`) |
 | **X-SAMPA** | Exact for all canonical symbols | Exact except aliases (`f\`→ɸ, `&`→æ) normalise to their canonical spelling | Passed through unchanged |
 | **Buckwalter ↔ Arabic** | Exact (the `^` shadda alias normalises to canonical `~`; precomposed lam-alef ligatures decompose to two chars, visually identical) | Exact | Passed through unchanged |
 | **Lexique ↔ IPA** | Exact except the `°`/`3` schwa pair (both → `ə`; reverse always → `°`) | Exact | Passed through unchanged |

{scriptconv-0.0.3a10 → scriptconv-0.0.3a12}/README.md

@@ -177,7 +177,7 @@ below describes the per-converter default.
 
 | Notation | `to_ipa` → `from_ipa` round-trip | `from_ipa` → `to_ipa` round-trip | Unknown-token behaviour |
 |----------|----------------------------------|----------------------------------|-------------------------|
-| **ARPABET** | Exact for base symbols; **stress digits are dropped** and `AH0`↔`AX` (schwa) is not distinguished | **Lossy** — ARPABET is an English-only inventory, so any IPA symbol outside it becomes the *unknown* placeholder | `arpa_to_ipa`: passed through unchanged. `ipa_to_arpa`: diacritics and suprasegmentals (combining marks, length/stress modifiers) are dropped; other out-of-inventory symbols are replaced with `?` by default (`unknown=` param; `unknown=""` drops) |
+| **ARPABET** | **Lossless with `stress=True`** (digits ↔ IPA `ˈ`/`ˌ` before the stressed vowel; residues: extended-ARPABET `AX` normalises to CMUdict's `AH0` spelling, and `AH0 R` fuses to the r-colored `AXR0` — stable from the IPA side). Default `stress=False` drops digits and merges `AH0`↔`AX` | **Lossy** — ARPABET is an English-only inventory, so any IPA symbol outside it becomes the *unknown* placeholder | `arpa_to_ipa`: passed through unchanged. `ipa_to_arpa`: diacritics and suprasegmentals are dropped; other out-of-inventory symbols follow `errors=` (default `"replace"` → `?`) |
 | **X-SAMPA** | Exact for all canonical symbols | Exact except aliases (`f\`→ɸ, `&`→æ) normalise to their canonical spelling | Passed through unchanged |
 | **Buckwalter ↔ Arabic** | Exact (the `^` shadda alias normalises to canonical `~`; precomposed lam-alef ligatures decompose to two chars, visually identical) | Exact | Passed through unchanged |
 | **Lexique ↔ IPA** | Exact except the `°`/`3` schwa pair (both → `ə`; reverse always → `°`) | Exact | Passed through unchanged |

{scriptconv-0.0.3a10 → scriptconv-0.0.3a12}/pyproject.toml

@@ -35,7 +35,25 @@ dependencies = {file = ["requirements.txt"]}
 [project.optional-dependencies]
 ja = ["pykakasi>=2.3,<3"]
 zh = ["pypinyin>=0.50,<1"]
-test = ["pytest", "pytest-timeout", "pykakasi>=2.3,<3", "pypinyin>=0.50,<1"]
+# --- phonemizer wrappers (scriptconv.phonemizers) ---
+phonemizers = ["quebra-frases", "langcodes"]
+espeak = ["scriptconv[phonemizers]", "espyak"]
+gruut = ["scriptconv[phonemizers]", "gruut"]
+goruut = ["scriptconv[phonemizers]", "pygoruut"]
+epitran = ["scriptconv[phonemizers]", "epitran"]
+transphone = ["scriptconv[phonemizers]", "transphone"]
+misaki = ["scriptconv[phonemizers]", "misaki"]
+byt5 = ["scriptconv[phonemizers]", "onnxruntime", "numpy"]
+o2i = ["scriptconv[phonemizers]", "orthography2ipa"]
+eu = ["scriptconv[phonemizers]", "euskaphone", "ahotts_g2p"]
+pt-phonemizers = ["scriptconv[phonemizers]", "tugaphone", "g2p_barranquenho"]
+mwl = ["scriptconv[phonemizers]", "mwl_phonemizer"]
+gl = ["scriptconv[phonemizers]", "pycotovia"]
+he = ["scriptconv[phonemizers]", "phonikud-onnx"]
+fa = ["scriptconv[phonemizers]", "persian_phonemizer"]
+vi = ["scriptconv[phonemizers]", "viphoneme"]
+tashkeel = ["scriptconv[phonemizers]", "text2tashkeel"]
+test = ["pytest", "pytest-timeout", "pykakasi>=2.3,<3", "pypinyin>=0.50,<1", "quebra-frases", "langcodes"]
 
 [project.urls]
 Homepage = "https://github.com/TigreGotico/scriptconv"

{scriptconv-0.0.3a10 → scriptconv-0.0.3a12}/scriptconv/notation.py

@@ -157,6 +157,11 @@ _IPA_TO_ARPA.setdefault("g", "G")
 _IPA_ARPA_KEYS_SORTED = sorted(_IPA_TO_ARPA.keys(), key=len, reverse=True)
 _IPA_ARPA_RE = re.compile("|".join(re.escape(s) for s in _IPA_ARPA_KEYS_SORTED))
 
+# ARPABET vowel class — the symbols that carry stress digits.
+_ARPA_VOWELS = {"AO", "AA", "IY", "UW", "EH", "IH", "UH", "AH", "AE", "AX",
+                "EY", "AY", "OW", "AW", "OY", "ER", "AXR"}
+_STRESS_MARKS = {"ˈ": "1", "ˌ": "2"}  # IPA primary/secondary stress
+
 
 
 class UnknownSymbolError(ValueError):
@@ -187,12 +192,18 @@ def _unknown(symbol: str, position: int, notation: str, errors: str,
     raise ValueError(f"errors must be one of {_ERROR_POLICIES}, not {errors!r}")
 
 
-def arpa_to_ipa(arpa_sequence: str, errors: str = "pass") -> str:
+def arpa_to_ipa(arpa_sequence: str, errors: str = "pass",
+                stress: bool = False) -> str:
     """Convert a space-separated ARPABET sequence to an IPA string.
 
-    Stress digits are stripped.  ``errors`` selects the unknown-token policy:
-    ``"pass"`` (default) keeps the token, ``"replace"`` substitutes ``?``,
-    ``"ignore"`` drops it, ``"strict"`` raises :class:`UnknownSymbolError`.
+    With ``stress=True``, ARPABET stress digits are preserved as IPA stress
+    marks placed immediately before the stressed vowel's symbol: ``AH1`` →
+    ``ˈʌ``, ``AH2`` → ``ˌʌ``, ``AH0`` → unmarked (a purely notational,
+    reversible convention — no syllabification is performed).  The default
+    ``stress=False`` keeps the historical behavior of stripping the digits.
+    ``errors`` selects the unknown-token policy: ``"pass"`` (default) keeps
+    the token, ``"replace"`` substitutes ``?``, ``"ignore"`` drops it,
+    ``"strict"`` raises :class:`UnknownSymbolError`.
 
     Examples
     --------
@@ -202,6 +213,9 @@ def arpa_to_ipa(arpa_sequence: str, errors: str = "pass") -> str:
     tokens = arpa_sequence.strip().split()
     result = []
     for i, tok in enumerate(tokens):
+        mark = ""
+        if stress and tok[-1:] in ("1", "2") and tok[:-1] in _ARPA_VOWELS:
+            mark = "ˈ" if tok[-1] == "1" else "ˌ"
         ipa = _ARPA_TO_IPA.get(tok)
         if ipa is None:
             # strip trailing digit and retry
@@ -209,19 +223,24 @@ def arpa_to_ipa(arpa_sequence: str, errors: str = "pass") -> str:
             ipa = _ARPA_TO_IPA.get(stripped)
         if ipa is None:
             ipa = _unknown(tok, i, "arpa", errors)
-        result.append(ipa)
+            mark = ""
+        result.append(mark + ipa)
     return "".join(result)
 
 
 def ipa_to_arpa(ipa_string: str, unknown: str = "?",
-                errors: str = "replace") -> str:
+                errors: str = "replace", stress: bool = False) -> str:
     """Convert an IPA string to a space-separated ARPABET sequence.
 
-    Matches longest IPA symbol first.  ``errors`` selects the unknown-symbol
-    policy (default ``"replace"``, preserving the historical behavior of
-    substituting *unknown*, ``"?"``); ``unknown=""`` still drops silently.
-    Combining marks and modifier letters qualify the preceding phoneme and
-    are always dropped rather than treated as unknowns.
+    Matches longest IPA symbol first.  With ``stress=True``, IPA stress
+    marks (``ˈ``/``ˌ``) preceding a vowel-class symbol are emitted as that
+    vowel's stress digit (``1``/``2``) and unmarked vowels receive ``0`` —
+    the reverse of :func:`arpa_to_ipa`'s ``stress=True`` convention.
+    ``errors`` selects the unknown-symbol policy (default ``"replace"``,
+    preserving the historical behavior of substituting *unknown*, ``"?"``);
+    ``unknown=""`` still drops silently.  Combining marks and modifier
+    letters qualify the preceding phoneme and are always dropped rather
+    than treated as unknowns.
 
     Examples
     --------
@@ -231,11 +250,25 @@ def ipa_to_arpa(ipa_string: str, unknown: str = "?",
     '?'
     """
     tokens = []
+    pending_stress = ""
     pos = 0
     while pos < len(ipa_string):
+        if stress and ipa_string[pos] in _STRESS_MARKS:
+            pending_stress = _STRESS_MARKS[ipa_string[pos]]
+            pos += 1
+            continue
         m = _IPA_ARPA_RE.match(ipa_string, pos)
         if m:
-            tokens.append(_IPA_TO_ARPA[m.group(0)])
+            tok = _IPA_TO_ARPA[m.group(0)]
+            if stress and tok in _ARPA_VOWELS:
+                # CMUdict spells schwa as AH0 (it has no AX), so stress-aware
+                # output uses AH for exact round-trips; extended-ARPABET AX
+                # input still normalizes to AH0 (documented residue).
+                if m.group(0) == "ə":
+                    tok = "AH"
+                tok += pending_stress or "0"
+            pending_stress = ""
+            tokens.append(tok)
             pos = m.end()
         else:
             ch = ipa_string[pos]

scriptconv-0.0.3a12/scriptconv/phonemizers/__init__.py

@@ -0,0 +1,40 @@
+"""Phonemizer wrappers — see :mod:`scriptconv.phonemizers.base` for the thesis.
+
+Public surface: the :class:`BasePhonemizer` contract, the ``Alphabet``/
+``Phonemizer`` enums, the lazy registry (``get_phonemizer``,
+``phonemizer_for_lang``), and the per-language wrapper modules.  Backing
+engines are optional extras; a missing one raises :class:`ImportError`
+naming the extra to install.
+"""
+from scriptconv.phonemizers.enums import Alphabet, Phonemizer
+from scriptconv.phonemizers.base import (
+    BasePhonemizer,
+    GraphemePhonemizer,
+    UnicodeCodepointPhonemizer,
+    TextChunks,
+    RawPhonemizedChunks,
+    PhonemizedChunks,
+)
+from scriptconv.phonemizers.registry import (
+    PHONEMIZER_REGISTRY,
+    LANG_DEFAULTS,
+    get_phonemizer,
+    get_phonemizer_class,
+    phonemizer_for_lang,
+)
+
+__all__ = [
+    "Alphabet",
+    "Phonemizer",
+    "BasePhonemizer",
+    "GraphemePhonemizer",
+    "UnicodeCodepointPhonemizer",
+    "TextChunks",
+    "RawPhonemizedChunks",
+    "PhonemizedChunks",
+    "PHONEMIZER_REGISTRY",
+    "LANG_DEFAULTS",
+    "get_phonemizer",
+    "get_phonemizer_class",
+    "phonemizer_for_lang",
+]

scriptconv-0.0.3a12/scriptconv/phonemizers/base.py

@@ -0,0 +1,332 @@
+"""Phonemizer contract and base implementations.
+
+Phonemization (orthography → sound) is out of scope for scriptconv's
+zero-dependency core; this subpackage holds *wrappers* around external
+phonemizer engines behind optional extras — the same boundary as
+:mod:`scriptconv.readings` wrapping reading dictionaries.  No
+grapheme-to-phoneme rules are implemented in the core; in-tree
+implementations live under ``_thirdparty`` with their provenance, installed
+only with the ``phonemizers`` extras.
+
+The ``phonemizers`` base extra provides sentence chunking (``quebra_frases``)
+and language matching (``langcodes``).
+
+**Normalization is injectable, not built in.**  TTS stacks normalize numbers,
+dates and abbreviations before phonemizing; that requires language resources
+scriptconv does not ship.  Pass ``normalizer=`` (a ``(text, lang) -> str``
+callable) at construction to run it inside ``phonemize_lazy``; without it the
+raw text is phonemized as-is.  Downstream TTS injects its own normalizer, so
+calling a wrapper through scriptconv directly and through a TTS stack can
+legitimately differ on text containing digits.
+"""
+import abc
+import re
+import string
+import unicodedata
+from typing import Callable, Iterator, List, Optional, Tuple, Literal, Union
+
+from scriptconv.phonemizers.enums import Alphabet
+
+# list of (substring, terminator, end_of_sentence) tuples.
+TextChunks = List[Tuple[str, str, bool]]
+# list of (phonemes, terminator, end_of_sentence) tuples.
+RawPhonemizedChunks = List[Tuple[str, str, bool]]
+
+PhonemizedChunks = list[list[str]]
+
+
+class BasePhonemizer(metaclass=abc.ABCMeta):
+    def __init__(self, alphabet: Alphabet = Alphabet.UNICODE,
+                 diacritizer_model: str = "rawi-ensemble",
+                 normalizer: Optional[Callable[[str, str], str]] = None,
+                 phonikud_model: Optional[str] = None):
+        super().__init__()
+        self.alphabet = alphabet
+        # optional (text, lang) -> str hook run before chunking; see module
+        # docstring — scriptconv performs no normalization of its own
+        self.normalizer = normalizer
+        # local path to a phonikud ONNX model (Hebrew diacritization);
+        # scriptconv never downloads — the consumer resolves the file
+        self.phonikud_model = phonikud_model
+
+        # diacritizer model name, for languages that need one. Arabic uses
+        # text2tashkeel; the default "rawi-ensemble" restores hamza and the dagger
+        # alef in addition to the standard marks.
+        self.diacritizer_model = diacritizer_model
+        self._phonikud = None  # hebrew only
+        self._tashkeel: dict = {}  # model name -> text2tashkeel Diacritizer
+
+    @property
+    def phonikud(self):
+        if self._phonikud is None:
+            if not self.phonikud_model:
+                raise ValueError(
+                    "Hebrew diacritization needs a local phonikud ONNX model: "
+                    "pass phonikud_model=<path> (scriptconv never downloads "
+                    "models; obtain one from the phonikud-onnx release)")
+            try:
+                from phonikud_onnx import Phonikud
+            except ImportError:
+                raise ImportError(
+                    "Hebrew diacritization needs phonikud-onnx — install "
+                    "with `pip install scriptconv[he]`") from None
+            self._phonikud = Phonikud(self.phonikud_model)
+        return self._phonikud
+
+    def tashkeel(self, model: Optional[str] = None):
+        """Lazily build (and cache) the text2tashkeel Diacritizer used for Arabic.
+
+        text2tashkeel is a dependency of the ``[ar]`` extra; it restores hamza and the
+        dagger alef in addition to the standard marks. Install with
+        ``pip install scriptconv[tashkeel]`` (or ``pip install text2tashkeel``)."""
+        model = model or self.diacritizer_model
+        if model not in self._tashkeel:
+            try:
+                from text2tashkeel import Diacritizer
+            except ImportError as e:
+                raise ImportError(
+                    "Arabic diacritization requires the text2tashkeel package: "
+                    "pip install scriptconv[tashkeel]  (or pip install text2tashkeel)"
+                ) from e
+            self._tashkeel[model] = Diacritizer(model)
+        return self._tashkeel[model]
+
+    @abc.abstractmethod
+    def phonemize_string(self, text: str, lang: str) -> str:
+        raise NotImplementedError
+
+    def phonemize_to_list(self, text: str, lang: str) -> List[str]:
+        return list(self.phonemize_string(text, lang))
+
+    def add_diacritics(self, text: str, lang: str,
+                       model: Optional[str] = None) -> str:
+        if lang.startswith("he"):
+            return self.phonikud.add_diacritics(text)
+        elif lang.startswith("ar"):
+            return self.tashkeel(model).diacritize(text)
+        return text
+
+    def phonemize(self, text: str, lang: str) -> PhonemizedChunks:
+        # PhonemizedChunks is list[list[str]]; empty text yields no
+        # sentences. (Returning the raw (str, str, bool) tuple form here
+        # corrupted the type and broke callers that mutate each sentence,
+        # e.g. inline ``[[phoneme]]`` blocks in TTSVoice.phonemize.)
+        return list(self.phonemize_lazy(text, lang))
+
+    def phonemize_lazy(self, text: str, lang: str) -> Iterator[List[str]]:
+        """Lazy, per-sentence variant of :meth:`phonemize`.
+
+        Yields one phoneme list per sentence-level chunk, invoking the
+        (potentially expensive) ``phonemize_string`` only as each sentence is
+        pulled from the generator. This lets a caller start synthesizing
+        sentence 1 before sentence 2 has been phonemized, cutting
+        time-to-first-audio.
+
+        Normalization and chunking still run over the *whole* text up front (both
+        are cheap and order-sensitive), so ``list(self.phonemize_lazy(text)) ==
+        self.phonemize(text)`` holds for any input.
+        """
+        if not text:
+            return
+        if self.normalizer is not None:
+            text = self.normalizer(text, lang)
+        for chunk, punct, eos in self.chunk_text(text):
+            phoneme_str = self.phonemize_string(self.remove_punctuation(chunk), lang)
+            # Filter out (lang) switch (flags) that surround words from languages
+            # other than the current voice — mirrors _process_phones().
+            phoneme_str = re.sub(r"\([^)]+\)", "", phoneme_str)
+            # phonemize() marks every chunk as end-of-sentence, so each chunk is
+            # its own sentence.
+            yield list(phoneme_str)
+
+    @staticmethod
+    def _process_phones(raw_phones: RawPhonemizedChunks) -> PhonemizedChunks:
+        """Text to phonemes grouped by sentence."""
+        all_phonemes: list[list[str]] = []
+        sentence_phonemes: list[str] = []
+        for phonemes_str, terminator_str, end_of_sentence in raw_phones:
+            # Filter out (lang) switch (flags).
+            # These surround words from languages other than the current voice.
+            phonemes_str = re.sub(r"\([^)]+\)", "", phonemes_str)
+            sentence_phonemes.extend(list(phonemes_str))
+            if end_of_sentence:
+                all_phonemes.append(sentence_phonemes)
+                sentence_phonemes = []
+        if sentence_phonemes:
+            all_phonemes.append(sentence_phonemes)
+        return all_phonemes
+
+    @staticmethod
+    def _match_lang(target_lang: str, valid_langs: Union[str, List[str]]):
+        """Closest supported language and its distance (langcodes tag_distance)."""
+        from langcodes import tag_distance
+        if isinstance(valid_langs, str):
+            valid_langs = [valid_langs]
+        if target_lang in valid_langs:
+            return target_lang, 0
+        best_lang, best_distance = "und", 10000000
+        for l in valid_langs:
+            try:
+                distance = tag_distance(l, target_lang)
+            except Exception:
+                try:
+                    distance = tag_distance(l.split("-")[0], target_lang)
+                except Exception:
+                    continue
+            if distance < best_distance:
+                best_lang, best_distance = l, distance
+        if best_distance <= 10:
+            return best_lang, best_distance
+        return "und", 10000
+
+    @staticmethod
+    def match_lang(target_lang: str, valid_langs: List[str]) -> str:
+        """
+        Validates and returns the closest supported language code.
+
+        Args:
+            target_lang (str): The language code to validate.
+
+        Returns:
+            str: The validated language code.
+
+        Raises:
+            ValueError: If the language code is unsupported.
+        """
+        lang, score = BasePhonemizer._match_lang(target_lang, valid_langs)
+        if score > 10:
+            # raise an error for unsupported language
+            raise ValueError(f"unsupported language code: {target_lang}")
+        return lang
+
+    @staticmethod
+    def remove_punctuation(text):
+        """
+        Removes punctuation characters from a string, based on unicode punctuation
+        categories (P*) rather than ASCII-only ``string.punctuation``. This also strips
+        non-ASCII punctuation such as Arabic ``،``/``؟`` and curly quotes.
+
+        Apostrophes and hyphens sandwiched between letters (e.g. "don't", "well-known")
+        are preserved so contractions and compounds aren't broken apart.
+        """
+        out = []
+        chars = list(text)
+        for i, c in enumerate(chars):
+            if c in ("'", "’", "-") and 0 < i < len(chars) - 1 \
+                    and chars[i - 1].isalpha() and chars[i + 1].isalpha():
+                out.append(c)
+                continue
+            if unicodedata.category(c).startswith("P"):
+                continue
+            out.append(c)
+        return "".join(out).strip()
+
+    @staticmethod
+    def chunk_text(text: str, delimiters: Optional[List[str]] = None) -> TextChunks:
+        """
+        Split input text into sentence-aware chunks using sentence tokenization and optional intra-sentence delimiters.
+
+        Parameters:
+            text (str): Input text to split. If empty, returns a single empty chunk.
+            delimiters (Optional[List[str]]): List of substring delimiters to split sentences by (defaults to [":", ";", "...", "|"]).
+
+        Returns:
+            TextChunks: A list of tuples (substring, terminator, end_of_sentence) where:
+                - `substring` is the chunk text with surrounding whitespace removed,
+                - `terminator` is the delimiter that followed the substring or the sentence-final punctuation if none matched,
+                - `end_of_sentence` is `True` for the final chunk of a tokenized sentence, `False` otherwise.
+        """
+        if not text:
+            return [('', '', True)]
+
+        try:
+            from quebra_frases import sentence_tokenize
+        except ImportError:
+            raise ImportError(
+                "sentence chunking needs quebra-frases — install with "
+                "`pip install scriptconv[phonemizers]`") from None
+
+        results: TextChunks = []
+        delimiters = delimiters or [":", ";", "...", "|"]
+
+        # Create a regex pattern that matches any of the delimiters
+        delimiter_pattern = re.escape(delimiters[0])
+        for delimiter in delimiters[1:]:
+            delimiter_pattern += f"|{re.escape(delimiter)}"
+
+        for sentence in sentence_tokenize(text):
+            # Default punctuation if no specific punctuation found
+            default_punc = sentence[-1] if sentence and sentence[-1] in string.punctuation else "."
+
+            # Use regex to split the sentence by any of the delimiters
+            parts = re.split(f'({delimiter_pattern})', sentence)
+
+            # Group parts into chunks (text + delimiter)
+            chunks = []
+            for i in range(0, len(parts), 2):
+                # If there's a delimiter after the text, use it
+                delimiter = parts[i + 1] if i + 1 < len(parts) else default_punc
+
+                # Last chunk is marked as complete
+                is_last = (i + 2 >= len(parts))
+
+                chunks.append((parts[i].strip(), delimiter.strip(), is_last))
+
+            results.extend(chunks)
+
+        return results
+
+
+class GraphemePhonemizer(BasePhonemizer):
+    """
+    A phonemizer class that treats input text as graphemes (characters).
+    It performs text normalization and returns the normalized text as a string
+    of characters.
+    """
+    # Regular expression matching whitespace:
+    whitespace_re = re.compile(r"\s+")
+
+    def phonemize_string(self, text: str, lang: str) -> str:
+        """
+        Normalizes input text by applying a series of transformations
+        and returns it as a sequence of graphemes.
+
+        Parameters:
+            text (str): Input text to be converted to graphemes.
+            lang (str): The language code (ignored for grapheme phonemization,
+                        but required by BasePhonemizer).
+
+        Returns:
+            str: A normalized string of graphemes.
+        """
+        text = text.lower()
+        text = text.replace(";", ",")
+        text = text.replace("-", " ")
+        text = text.replace(":", ",")
+        text = re.sub(r"[\<\>\(\)\[\]\"]+", "", text)
+        text = re.sub(self.whitespace_re, " ", text).strip()
+        return text
+
+
+class UnicodeCodepointPhonemizer(BasePhonemizer):
+    """Phonemes = codepoints
+    normalization also splits accents and punctuation into it's own codepoints
+    """
+
+    def __init__(self, form: Literal["NFC", "NFD", "NFKC", "NFKD"] = "NFD"):
+        self.form = form
+        super().__init__(Alphabet.UNICODE)
+
+    def phonemize_string(self, text: str, lang: str) -> str:
+        # Phonemes = codepoints
+        """
+        Normalize the input text to Unicode NFD so phonemes correspond to individual Unicode codepoints.
+
+        Parameters:
+            text (str): The input string to normalize.
+            lang (str): Language code (ignored by this implementation).
+
+        Returns:
+            str: The input text normalized to Unicode NFD, with combining marks separated into distinct codepoints.
+        """
+        return unicodedata.normalize(self.form, text)