scriptconv 0.0.4a4__tar.gz → 0.0.4a5__tar.gz

{scriptconv-0.0.4a4/scriptconv.egg-info → scriptconv-0.0.4a5}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: scriptconv
-Version: 0.0.4a4
+Version: 0.0.4a5
 Summary: Zero-dependency script & phoneme-notation core — ISO-15924 detection & metadata, IPA↔ARPABET/X-SAMPA/Lexique/Kirshenbaum/Cotovía/RFE, Buckwalter↔Arabic, Hangul→jamo, kana
 License: Apache-2.0
 Project-URL: Homepage, https://github.com/TigreGotico/scriptconv
@@ -105,6 +105,7 @@ Requires-Dist: scriptconv[phonemizers]; extra == "en-phonemizers"
 Requires-Dist: g2p_en; extra == "en-phonemizers"
 Requires-Dist: openphonemizer; extra == "en-phonemizers"
 Requires-Dist: deep-phonemizer; extra == "en-phonemizers"
+Requires-Dist: requests; extra == "en-phonemizers"
 Provides-Extra: ar-phonemizers
 Requires-Dist: scriptconv[phonemizers]; extra == "ar-phonemizers"
 Requires-Dist: arbtok; extra == "ar-phonemizers"

{scriptconv-0.0.4a4 → scriptconv-0.0.4a5}/pyproject.toml

@@ -58,7 +58,7 @@ pt = ["scriptconv[phonemizers]", "bifonia>=0.1.1"]
 ja-phonemizers = ["scriptconv[phonemizers]", "pyopenjtalk", "cutlet", "pykakasi>=2.3,<3"]
 ko = ["scriptconv[phonemizers]", "g2pk", "regex"]
 zh-phonemizers = ["scriptconv[phonemizers]", "jieba", "pypinyin>=0.50,<1", "xpinyin", "g2pM", "pinyin_to_ipa"]
-en-phonemizers = ["scriptconv[phonemizers]", "g2p_en", "openphonemizer", "deep-phonemizer"]
+en-phonemizers = ["scriptconv[phonemizers]", "g2p_en", "openphonemizer", "deep-phonemizer", "requests"]
 ar-phonemizers = ["scriptconv[phonemizers]", "arbtok", "ovos-number-parser"]
 shami = ["scriptconv[phonemizers]", "regex"]
 test = ["pytest", "pytest-timeout", "pykakasi>=2.3,<3", "pypinyin>=0.50,<1", "quebra-frases", "langcodes", "regex", "ovos-number-parser", "bifonia>=0.1.1"]

{scriptconv-0.0.4a4 → scriptconv-0.0.4a5}/scriptconv/__init__.py

@@ -69,6 +69,7 @@ from scriptconv.notation import (
     ipa_to_cotovia,
     rfe_to_ipa,
     ipa_to_rfe,
+    mantoq_to_ipa,
     looks_like_ipa,
 )
 from scriptconv.translit import (
@@ -140,6 +141,7 @@ __all__ = [
     "ipa_to_cotovia",
     "rfe_to_ipa",
     "ipa_to_rfe",
+    "mantoq_to_ipa",
     "looks_like_ipa",
     # translit
     "decompose_hangul",

{scriptconv-0.0.4a4 → scriptconv-0.0.4a5}/scriptconv/cangjie.py

@@ -15,7 +15,7 @@ from __future__ import annotations
 
 import gzip
 from importlib.resources import files
-from typing import Dict, Optional
+from typing import Dict, List, Optional
 
 __all__ = ["cangjie_code", "to_cangjie"]
 
@@ -49,15 +49,16 @@ def to_cangjie(text: str, sep: str = " ") -> str:
     """
     table = _table()
     tokens = []
-    last_raw = False
+    raw_run: List[str] = []
     for ch in text:
         code = table.get(ch)
         if code is not None:
+            if raw_run:
+                tokens.append("".join(raw_run))
+                raw_run = []
             tokens.append(code)
-            last_raw = False
-        elif last_raw:
-            tokens[-1] += ch
         else:
-            tokens.append(ch)
-            last_raw = True
+            raw_run.append(ch)
+    if raw_run:
+        tokens.append("".join(raw_run))
     return sep.join(tokens)

{scriptconv-0.0.4a4 → scriptconv-0.0.4a5}/scriptconv/diacritics.py

@@ -45,7 +45,7 @@ tashkeel, stressonnx) are strippable; spelling-integral backends (bifonia) are
 not.
 """
 from scriptconv.graph import Edge
-from scriptconv.phonemizers.base import STRESS_LANGS, _primary_subtag
+from scriptconv.phonemizers.base import _diacritizer_family
 
 #: The lang-contextual node produced by diacritization.  Like ``"text"`` it is
 #: meaningful only with ``lang=`` context and exists only in opted-in graphs.
@@ -55,6 +55,9 @@ _DEFAULT_PHONEMIZER = None
 
 
 def _get_phonemizer(phonikud_model=None):
+    # any concrete BasePhonemizer subclass works here — it's just a vessel
+    # for add_diacritics(); GraphemePhonemizer is picked for having no extra
+    # runtime dependencies of its own
     global _DEFAULT_PHONEMIZER
     if phonikud_model:
         from scriptconv.phonemizers.base import GraphemePhonemizer
@@ -81,6 +84,10 @@ def diacritize(text: str, lang: str = "und", model=None, **context) -> str:
 # which (via NFD) would decompose and destroy precomposed native letters:
 # Cyrillic й/ё, Latvian ī, Azerbaijani ç/ö, and Arabic hamza carriers أ إ آ ؤ ئ.
 _STRESS_MARKS = frozenset({0x0300, 0x0301})            # combining grave / acute
+# U+0300 is included defensively alongside the U+0301 that stressonnx
+# actually emits, in case any backend/locale marks secondary stress with a
+# grave instead of an acute; harmless to strip since native precomposed
+# letters (e.g. Cyrillic й/ё) are unaffected either way.
 _ARABIC_MARKS = frozenset(range(0x064B, 0x0660)) | frozenset({0x0670})  # tashkeel + dagger alef
 _HEBREW_MARKS = (frozenset(range(0x05B0, 0x05BE))
                  | frozenset({0x05BF, 0x05C1, 0x05C2, 0x05C4, 0x05C5, 0x05C7}))  # niqqud
@@ -89,16 +96,18 @@ _HEBREW_MARKS = (frozenset(range(0x05B0, 0x05BE))
 def _overlay_marks(lang: str) -> "frozenset | None":
     """The overlay codepoints for *lang*'s diacritization backend, or None.
 
-    Uses exact primary-subtag matching (never ``startswith``) so Aragonese
-    (``arg``), Herero (``her``), Mapudungun (``arn``) etc. are NOT misread as
-    Arabic/Hebrew and stripped.
+    Resolves the backend family through :func:`_diacritizer_family` (the shared
+    lang→backend routing), so this can never disagree with
+    :meth:`BasePhonemizer.add_diacritics` about which language a backend owns.
+    Only *overlay* families are strippable: ``"pt"`` (bifonia sense marks are
+    native orthography) and ``None`` both return None.
     """
-    p = _primary_subtag(lang)
-    if p in STRESS_LANGS:
+    family = _diacritizer_family(lang)
+    if family == "stress":
         return _STRESS_MARKS
-    if p == "ar":
+    if family == "ar":
         return _ARABIC_MARKS
-    if p == "he":
+    if family == "he":
         return _HEBREW_MARKS
     return None
 

{scriptconv-0.0.4a4 → scriptconv-0.0.4a5}/scriptconv/graph.py

@@ -38,6 +38,11 @@ from typing import Callable, Dict, List, Optional, Tuple
 __all__ = ["Representation", "Edge", "ConversionGraph", "DEFAULT_GRAPH",
            "REPRESENTATIONS"]
 
+# Chosen so a single lossy hop never wins over a lossless path unless the
+# lossless alternative is at least this many hops longer. A 10-hop lossless
+# chain (cost 10, unit edges) ties a 1-hop lossy edge (cost 10) exactly, and
+# heapq's stable tie-break then decides — a non-issue in practice since real
+# scriptconv graphs are shallow (a handful of hops at most).
 _LOSSY_COST = 10.0
 
 
@@ -56,7 +61,10 @@ class Edge:
     """One registered transform between two representations.
 
     ``fn`` is called as ``fn(text, **context)``; context keys (``lang``,
-    engine-specific options…) pass through the router opaquely.  ``requires``
+    engine-specific options…) pass through the router opaquely.  Every
+    registered ``fn`` MUST accept ``**kwargs`` for this reason — a callable
+    that only takes ``text`` raises :class:`TypeError` the moment routing
+    passes through any context. ``requires``
     names an optional extra the transform needs — metadata only; the
     transform itself raises :class:`ImportError` with an install hint.
     ``cost`` defaults from ``lossless`` so routing prefers lossless paths.

{scriptconv-0.0.4a4 → scriptconv-0.0.4a5}/scriptconv/notation.py

@@ -41,6 +41,7 @@ __all__ = [
     "ipa_to_cotovia",
     "rfe_to_ipa",
     "ipa_to_rfe",
+    "mantoq_to_ipa",
     "looks_like_ipa",
 ]
 
@@ -1053,7 +1054,7 @@ def _tokenize_mantoq(text: str) -> list[str]:
     return tokens
 
 
-def mantoq_to_ipa(mantoq, errors: str = "pass") -> str:
+def mantoq_to_ipa(mantoq: str | list[str], errors: str = "pass") -> str:
     """Convert a Mantoq phoneme string to IPA.
 
     ``_dbl_`` lengthens/geminates the preceding symbol (``ː``), ``_+_``
@@ -1235,15 +1236,19 @@ def can_convert(src: str | Notation, dst: str | Notation) -> bool:
     src = Notation(src)
     dst = Notation(dst)
     if src == dst:
-        # historical contract: identity is not a "conversion"
+        # historical contract: identity is not a "conversion" here, unlike
+        # graph.ConversionGraph.can_convert (which deliberately returns True
+        # for identity — an empty route is a valid, zero-cost conversion there)
         return False
     from scriptconv.graph import DEFAULT_GRAPH
-    if not DEFAULT_GRAPH.can_convert(src.value, dst.value):
+    try:
+        route = DEFAULT_GRAPH.route(src.value, dst.value)
+    except ValueError:
         return False
     # only notation-to-notation reachability counts here; guard against paths
     # that would leave the notation node set (none exist today, cheap to keep)
     return all(e.src in _NOTATION_VALUES and e.dst in _NOTATION_VALUES
-               for e in DEFAULT_GRAPH.route(src.value, dst.value))
+               for e in route)
 
 
 # ---------------------------------------------------------------------------

{scriptconv-0.0.4a4 → scriptconv-0.0.4a5}/scriptconv/phonemizers/_thirdparty/hangul2ipa.py

@@ -2,9 +2,10 @@
 import csv
 import math
 import os.path
+import threading
 from base64 import b64decode
 from pathlib import Path
-from typing import Union, List, Dict
+from typing import Union, List, Dict, Optional
 
 import regex as re
 
@@ -311,21 +312,46 @@ OBSTRUENTS = ()
 SONORANTS = ()
 
 
+_initialized_tables_dir: Optional[Path] = None
+# guards the one-time table population below. The rule functions read the
+# module globals (CT_*, CONSONANTS, VOWELS, ...) without locking, which is safe
+# only because those globals are written exactly once and never mutated
+# afterwards; this lock serialises concurrent first-time initialisation so no
+# thread can observe a half-populated set of tables.
+_init_lock = threading.Lock()
+
+
 def initialize_conversion_tables(tables_dir: Path):
-    global CT_double_codas, CT_neutral, CT_tensification, CT_assimilation, CT_aspiration, CT_convention
-    CT_double_codas = ConversionTable('double_coda', tables_dir)
-    CT_neutral = ConversionTable('neutralization', tables_dir)
-    CT_tensification = ConversionTable('tensification', tables_dir)
-    CT_assimilation = ConversionTable('assimilation', tables_dir)
-    CT_aspiration = ConversionTable('aspiration', tables_dir)
-    CT_convention = ConversionTable('ipa', tables_dir)
-
-    global CONSONANTS, VOWELS, OBSTRUENTS, SONORANTS
-    CONSONANTS = tuple(
-        list(CT_convention.C)[:-2])  # from the C column of the IPA table, remove special characters # and $
-    VOWELS = tuple(list(CT_convention.V))  # from the V column of the IPA table
-    OBSTRUENTS = tuple(set(CONSONANTS) - set(C_SONORANTS))
-    SONORANTS = VOWELS + C_SONORANTS
+    global _initialized_tables_dir
+    # the CSV tables never change at runtime, so once a given tables_dir has
+    # been parsed there is no need to re-read and re-parse it on every call.
+    # Fast path is lock-free: _initialized_tables_dir is published (assigned)
+    # last, after every table global is fully populated, so any thread that
+    # sees it set also sees the complete tables.
+    if _initialized_tables_dir == tables_dir:
+        return
+    with _init_lock:
+        # re-check under the lock: another thread may have initialised while we
+        # waited (double-checked locking)
+        if _initialized_tables_dir == tables_dir:
+            return
+        global CT_double_codas, CT_neutral, CT_tensification, CT_assimilation, CT_aspiration, CT_convention
+        CT_double_codas = ConversionTable('double_coda', tables_dir)
+        CT_neutral = ConversionTable('neutralization', tables_dir)
+        CT_tensification = ConversionTable('tensification', tables_dir)
+        CT_assimilation = ConversionTable('assimilation', tables_dir)
+        CT_aspiration = ConversionTable('aspiration', tables_dir)
+        CT_convention = ConversionTable('ipa', tables_dir)
+
+        global CONSONANTS, VOWELS, OBSTRUENTS, SONORANTS
+        CONSONANTS = tuple(
+            list(CT_convention.C)[:-2])  # from the C column of the IPA table, remove special characters # and $
+        VOWELS = tuple(list(CT_convention.V))  # from the V column of the IPA table
+        OBSTRUENTS = tuple(set(CONSONANTS) - set(C_SONORANTS))
+        SONORANTS = VOWELS + C_SONORANTS
+
+        # published last: any thread seeing this set also sees all tables above
+        _initialized_tables_dir = tables_dir
 
 
 def get_substring_ind(string: str, pattern: str) -> List[int]:

{scriptconv-0.0.4a4 → scriptconv-0.0.4a5}/scriptconv/phonemizers/_vendored/kog2p/__init__.py

@@ -33,6 +33,7 @@ import re
 import math
 import sys
 import optparse
+from functools import lru_cache
 
 # Option
 # Import-time CLI parsing removed for library use (the upstream script
@@ -366,10 +367,19 @@ kog2p_to_hangul = {
 }
 
 
+@lru_cache(maxsize=None)
+def _cachedReadRules(pver, rule_book):
+    # rulebook.txt never changes at runtime, so parse it once per path
+    # instead of re-opening and re-parsing it on every runKoG2P call
+    rule_in, rule_out = readRules(pver, rule_book)
+    return tuple(rule_in), tuple(rule_out)
+
+
 def runKoG2P(graph, rulebook=None):
     if not rulebook:
         rulebook = f"{os.path.dirname(__file__)}/rulebook.txt"
-    [rule_in, rule_out] = readRules(ver_info[0], rulebook)
+    rule_in, rule_out = _cachedReadRules(ver_info[0], rulebook)
+    rule_in, rule_out = list(rule_in), list(rule_out)
     words = graph.split()
     phonemized = []
     for w in words:

{scriptconv-0.0.4a4 → scriptconv-0.0.4a5}/scriptconv/phonemizers/base.py

@@ -74,6 +74,30 @@ def _is_european_portuguese(lang: str) -> bool:
     return norm == "pt" or norm == "pt-pt"
 
 
+def _diacritizer_family(lang: str) -> Optional[str]:
+    """Which diacritization backend family handles *lang*, or ``None``.
+
+    Single source of truth for lang→backend routing: both the forward
+    dispatch (:meth:`BasePhonemizer.add_diacritics`) and the strip direction
+    (:func:`scriptconv.diacritics._overlay_marks`) resolve through this, so the
+    two can never disagree about which language uses which backend. Returns one
+    of ``"he"`` (niqqud), ``"ar"`` (tashkeel), ``"stress"`` (stressonnx), ``"pt"``
+    (bifonia sense diacritics), or ``None``. Uses exact primary-subtag matching
+    (never ``startswith``), so Aragonese (``arg``), Herero (``her``) and
+    Mapudungun (``arn``) are never misread as Arabic/Hebrew.
+    """
+    p = _primary_subtag(lang)
+    if p == "he":
+        return "he"
+    if p == "ar":
+        return "ar"
+    if p in STRESS_LANGS:
+        return "stress"
+    if _is_european_portuguese(lang):
+        return "pt"
+    return None
+
+
 class BasePhonemizer(metaclass=abc.ABCMeta):
     def __init__(self, alphabet: Alphabet = Alphabet.UNICODE,
                  diacritizer_model: str = "rawi-ensemble",
@@ -199,13 +223,14 @@ class BasePhonemizer(metaclass=abc.ABCMeta):
         ``ImportError`` naming its extra when the optional dependency is
         missing — scriptconv never installs anything on the caller's behalf.
         """
-        if _primary_subtag(lang) == "he":
+        family = _diacritizer_family(lang)
+        if family == "he":
             return self.phonikud.add_diacritics(text)
-        elif _primary_subtag(lang) == "ar":
+        if family == "ar":
             return self.tashkeel(model).diacritize(text)
-        elif _primary_subtag(lang) in STRESS_LANGS:
+        if family == "stress":
             return self._stress(text, lang, model)
-        elif _is_european_portuguese(lang):
+        if family == "pt":
             return self._sense_diacritics_pt(text)
         return text
 

{scriptconv-0.0.4a4 → scriptconv-0.0.4a5}/scriptconv/phonemizers/en.py

@@ -1,7 +1,3 @@
-import os
-
-import requests
-
 from scriptconv.notation import _ARPA_TO_IPA as arpa_to_ipa_lookup
 from scriptconv.phonemizers.base import BasePhonemizer
 from scriptconv.phonemizers.enums import Alphabet
@@ -18,6 +14,8 @@ class DeepPhonemizer(BasePhonemizer):
     }
 
     def __init__(self, model="latin_ipa_forward.pt"):
+        import os
+
         import dp
         from dp.phonemizer import Phonemizer
         import torch
@@ -38,6 +36,7 @@ class DeepPhonemizer(BasePhonemizer):
                 os.makedirs(cache_dir, exist_ok=True)
                 model_path = os.path.join(cache_dir, model)
                 if not os.path.isfile(model_path):
+                    import requests
                     print(f"Downloading {model} from {url}...")
                     with requests.get(url, stream=True) as r:
                         r.raise_for_status()

{scriptconv-0.0.4a4 → scriptconv-0.0.4a5}/scriptconv/phonemizers/fa.py

@@ -5,7 +5,14 @@ from scriptconv.phonemizers.enums import Alphabet
 class PersianPhonemizer(BasePhonemizer):
     """https://github.com/de-mh/persian_phonemizer"""
     def __init__(self, alphabet=Alphabet.IPA):
-        from persian_phonemizer import Phonemizer
+        try:
+            from persian_phonemizer import Phonemizer
+        except ImportError as e:
+            raise ImportError(
+                "persian_phonemizer is required for the Persian phonemizer. "
+                "Install it with 'pip install persian_phonemizer' "
+                "(or 'pip install scriptconv[fa]')."
+            ) from e
         assert alphabet in [Alphabet.ERAAB, Alphabet.IPA]
         output_format = "IPA" if alphabet == Alphabet.IPA else 'eraab'
         self.g2p = Phonemizer(output_format)

{scriptconv-0.0.4a4 → scriptconv-0.0.4a5}/scriptconv/phonemizers/gl.py

@@ -10,7 +10,8 @@ class CotoviaPhonemizer(BasePhonemizer):
     G2P engine that has verified parity with the original C binary.
 
     Output alphabets:
-    - ``Alphabet.COTOVIA`` — raw Cotovia phoneme notation (e.g. ``"Este e uN ..."``)
+    - ``Alphabet.COTOVIA`` (default) — raw Cotovia phoneme notation (e.g.
+      ``"Este e uN ..."``), the binary-parity-tested native phoneset
     - ``Alphabet.IPA`` — IPA string produced by pycotovia's ``cotovia_to_ipa``
 
     Voices trained on Cotovia-alphabet output continue to receive the same
@@ -21,11 +22,12 @@ class CotoviaPhonemizer(BasePhonemizer):
     lazily so that ``import scriptconv.phonemizers`` works without it installed.
     """
 
-    def __init__(self, alphabet: Alphabet = Alphabet.IPA,
+    def __init__(self, alphabet: Alphabet = Alphabet.COTOVIA,
                  model: Optional[str] = None):
         """
         Args:
-            alphabet (Alphabet): ``COTOVIA`` (raw notation) or ``IPA``.
+            alphabet (Alphabet): ``COTOVIA`` (raw notation, the default) or
+                ``IPA``.
             model (Optional[str]): phonemizer variant, from the voice's
                 ``phonemizer_model``.  ``"stress"`` emits the cotovia notation
                 with the stressed vowel marked by a trailing ``^`` (e.g.

{scriptconv-0.0.4a4 → scriptconv-0.0.4a5}/scriptconv/phonemizers/he.py

@@ -5,7 +5,14 @@ from scriptconv.phonemizers.base import BasePhonemizer
 class PhonikudPhonemizer(BasePhonemizer):
 
     def __init__(self):
-        from phonikud import phonemize
+        try:
+            from phonikud import phonemize
+        except ImportError as e:
+            raise ImportError(
+                "phonikud is required for the Hebrew phonemizer. "
+                "Install it with 'pip install phonikud-onnx' "
+                "(or 'pip install scriptconv[he]')."
+            ) from e
         self.g2p = phonemize
         super().__init__(Alphabet.IPA)
 

{scriptconv-0.0.4a4 → scriptconv-0.0.4a5}/scriptconv/phonemizers/registry.py

@@ -17,6 +17,7 @@ from __future__ import annotations
 import importlib
 from typing import Dict, Optional, Tuple
 
+from scriptconv.phonemizers.base import _primary_subtag
 from scriptconv.phonemizers.enums import Alphabet, Phonemizer
 
 __all__ = ["PHONEMIZER_REGISTRY", "LANG_DEFAULTS", "get_phonemizer",
@@ -184,7 +185,7 @@ def phonemizer_for_lang(lang: str, alphabet: Alphabet = Alphabet.IPA,
     """
     if override is not None:
         return get_phonemizer(override, alphabet, model, **kwargs)
-    key = lang.replace("_", "-").split("-")[0].lower()
+    key = _primary_subtag(lang)
     explicit = LANG_DEFAULTS.get(key, ())
     for candidate in explicit:
         if alphabet in _EMITS.get(candidate, (Alphabet.IPA,)):

{scriptconv-0.0.4a4 → scriptconv-0.0.4a5}/scriptconv/phonemizers/zh.py

@@ -1,4 +1,5 @@
 import abc
+import unicodedata
 from typing import List
 
 
@@ -46,7 +47,9 @@ class JiebaPhonemizer(BasePhonemizer):
         import jieba
         lang = self.get_lang(lang)
         seg_list = jieba.cut(text, cut_all=False)
-        seg_list = [num2str(w) if w.isdigit() else w for w in seg_list]
+        # NFKC folds full-width digits (e.g. "３") to ASCII so num2str's
+        # DIGITS table (keyed by ASCII "0"-"9") never raises KeyError on them
+        seg_list = [num2str(unicodedata.normalize("NFKC", w)) if w.isdigit() else w for w in seg_list]
         return " ".join(seg_list)
 
 
@@ -68,7 +71,13 @@ class BaseChinesePinyinPhonemizer(BasePhonemizer):
         super().__init__(alphabet)
         self.jieba = jieba
         self.retone = retone
-        from pinyin_to_ipa import pinyin_to_ipa
+        try:
+            from pinyin_to_ipa import pinyin_to_ipa
+        except ImportError:
+            raise ImportError(
+                "pinyin->IPA conversion needs pinyin_to_ipa — install with "
+                "`pip install scriptconv[zh-phonemizers]`"
+            ) from None
         self.pinyin_to_ipa = pinyin_to_ipa
 
     @classmethod
@@ -95,7 +104,12 @@ class BaseChinesePinyinPhonemizer(BasePhonemizer):
         p = p.replace('˥˩', '↘')  # fourth tone
         p = p.replace('˥', '→')  # first tone
         p = p.replace(chr(635) + chr(809), 'ɨ').replace(chr(633) + chr(809), 'ɨ')
-        assert chr(809) not in p, p
+        # U+0329 (combining vertical line below, syllabic diacritic) is only
+        # ever expected attached to ɻ/ɹ (U+027B/U+0279) above, which the two
+        # replacements just consumed; any leftover means pinyin_to_ipa
+        # emitted an unanticipated syllabic marking. Rather than crashing in
+        # production, leave it in place — it's still valid IPA, just not
+        # collapsed to the ɨ shorthand.
         return p
 
     def to_ipa(self, phones: List[str]) -> List[str]:
@@ -132,7 +146,9 @@ class BaseChinesePinyinPhonemizer(BasePhonemizer):
             import jieba
             for chunk in jieba.cut(text, cut_all=False):
                 if chunk.isdigit():
-                    chunk = num2str(chunk)
+                    # NFKC folds full-width digits (e.g. "３") to ASCII so
+                    # num2str's DIGITS table never raises KeyError on them
+                    chunk = num2str(unicodedata.normalize("NFKC", chunk))
                 phones += self.get_pinyin(chunk)
                 phones += [" "]  # keep jieba whitespace
         else:
@@ -176,7 +192,13 @@ class G2pCPhonemizer(BaseChinesePinyinPhonemizer):
     """
 
     def __init__(self, alphabet=Alphabet.PINYIN, jieba: bool = True):
-        from g2pc import G2pC
+        try:
+            from g2pc import G2pC
+        except ImportError:
+            raise ImportError(
+                "g2pc phonemization needs g2pc — install with "
+                "`pip install scriptconv[zh-phonemizers]`"
+            ) from None
         self.g2p = G2pC()
         super().__init__(alphabet, jieba)
 
@@ -200,7 +222,13 @@ class G2pMPhonemizer(BaseChinesePinyinPhonemizer):
     """
 
     def __init__(self, alphabet=Alphabet.PINYIN, tone: bool = True, char_split: bool = False, jieba: bool = True):
-        from g2pM import G2pM
+        try:
+            from g2pM import G2pM
+        except ImportError:
+            raise ImportError(
+                "g2pM phonemization needs g2pM — install with "
+                "`pip install scriptconv[zh-phonemizers]`"
+            ) from None
         self.g2p = G2pM()
         self.tone = tone
         self.char_split = char_split
@@ -225,7 +253,13 @@ class XpinyinPhonemizer(BaseChinesePinyinPhonemizer):
     """
 
     def __init__(self, alphabet=Alphabet.PINYIN, tone_marks: str = "numbers", jieba: bool = True):
-        from xpinyin import Pinyin
+        try:
+            from xpinyin import Pinyin
+        except ImportError:
+            raise ImportError(
+                "xpinyin phonemization needs xpinyin — install with "
+                "`pip install scriptconv[zh-phonemizers]`"
+            ) from None
         self.g2p = Pinyin()
         self.tone_marks = tone_marks
         super().__init__(alphabet, jieba)
@@ -249,7 +283,13 @@ class PypinyinPhonemizer(BaseChinesePinyinPhonemizer):
     """
 
     def __init__(self, alphabet=Alphabet.PINYIN, jieba: bool = True):
-        from pypinyin import pinyin
+        try:
+            from pypinyin import pinyin
+        except ImportError:
+            raise ImportError(
+                "pypinyin phonemization needs pypinyin — install with "
+                "`pip install scriptconv[zh]`"
+            ) from None
         self.g2p = pinyin
         super().__init__(alphabet, jieba)
 

{scriptconv-0.0.4a4 → scriptconv-0.0.4a5}/scriptconv/scripts.py

@@ -416,6 +416,10 @@ def char_script(ch: str) -> Optional[str]:
     if len(ch) != 1:
         return None
     cp = ord(ch)
+    # Only the interval immediately at-or-before cp (idx) is ever checked —
+    # correct only because SCRIPT_REGISTRY's char_ranges are non-overlapping,
+    # so at most one interval can ever contain cp. If ranges ever overlapped,
+    # this single-candidate check would silently miss matches.
     idx = bisect_right(_SORTED_STARTS, cp) - 1
     if idx >= 0 and cp <= _SORTED_ENDS[idx]:
         return _SORTED_SCRIPTS[idx]

{scriptconv-0.0.4a4 → scriptconv-0.0.4a5}/scriptconv/version.py

@@ -2,7 +2,7 @@
 VERSION_MAJOR = 0
 VERSION_MINOR = 0
 VERSION_BUILD = 4
-VERSION_ALPHA = 4
+VERSION_ALPHA = 5
 # END_VERSION_BLOCK
 
 VERSION_STR = f"{VERSION_MAJOR}.{VERSION_MINOR}.{VERSION_BUILD}"

{scriptconv-0.0.4a4 → scriptconv-0.0.4a5/scriptconv.egg-info}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: scriptconv
-Version: 0.0.4a4
+Version: 0.0.4a5
 Summary: Zero-dependency script & phoneme-notation core — ISO-15924 detection & metadata, IPA↔ARPABET/X-SAMPA/Lexique/Kirshenbaum/Cotovía/RFE, Buckwalter↔Arabic, Hangul→jamo, kana
 License: Apache-2.0
 Project-URL: Homepage, https://github.com/TigreGotico/scriptconv
@@ -105,6 +105,7 @@ Requires-Dist: scriptconv[phonemizers]; extra == "en-phonemizers"
 Requires-Dist: g2p_en; extra == "en-phonemizers"
 Requires-Dist: openphonemizer; extra == "en-phonemizers"
 Requires-Dist: deep-phonemizer; extra == "en-phonemizers"
+Requires-Dist: requests; extra == "en-phonemizers"
 Provides-Extra: ar-phonemizers
 Requires-Dist: scriptconv[phonemizers]; extra == "ar-phonemizers"
 Requires-Dist: arbtok; extra == "ar-phonemizers"

{scriptconv-0.0.4a4 → scriptconv-0.0.4a5}/scriptconv.egg-info/SOURCES.txt

@@ -78,6 +78,7 @@ tests/test_arpa_stress.py
 tests/test_cangjie.py
 tests/test_cli.py
 tests/test_conventions.py
+tests/test_diacritics.py
 tests/test_diacritics_graph.py
 tests/test_errors_policy.py
 tests/test_examples.py

{scriptconv-0.0.4a4 → scriptconv-0.0.4a5}/scriptconv.egg-info/requires.txt

@@ -14,6 +14,7 @@ scriptconv[phonemizers]
 g2p_en
 openphonemizer
 deep-phonemizer
+requests
 
 [epitran]
 scriptconv[phonemizers]

scriptconv-0.0.4a5/tests/test_diacritics.py

@@ -0,0 +1,90 @@
+"""Pure-python tests for scriptconv.diacritics strip/overlay helpers.
+
+No optional backend is required — strip_diacritics/_overlay_marks/_supports_strip
+never load a phonemizer model.
+"""
+import pytest
+
+from scriptconv.diacritics import (
+    strip_diacritics,
+    _overlay_marks,
+    _supports_strip,
+    _STRESS_MARKS,
+    _ARABIC_MARKS,
+    _HEBREW_MARKS,
+)
+from scriptconv.phonemizers.base import STRESS_LANGS
+
+
+def test_strip_removes_combining_acute_preserves_cyrillic():
+    assert strip_diacritics("мой родно́й край", "ru") == "мой родной край"
+
+
+def test_strip_preserves_precomposed_cyrillic_letters():
+    # й (U+0439) and ё (U+0451) are precomposed, not combining marks — they
+    # must survive stripping even alongside a combining acute.
+    text = "ёж, война́, май"  # combining acute (U+0301) after "а" in "война́"
+    stripped = strip_diacritics(text, "ru")
+    assert "ё" in stripped
+    assert "й" in stripped
+    assert stripped == "ёж, война, май"
+
+
+def test_strip_arabic_removes_tashkeel_preserves_consonants():
+    assert strip_diacritics("مُحَمَّد", "ar") == "محمد"
+
+
+def test_strip_arabic_preserves_hamza_carrier():
+    stripped = strip_diacritics("أَحْمَد", "ar")
+    assert stripped.startswith("أ")
+
+
+def test_strip_hebrew_removes_niqqud_preserves_consonants():
+    # שָׁלוֹם with niqqud -> bare consonants שלום
+    stripped = strip_diacritics("שָׁלוֹם", "he")
+    assert stripped == "שלום"
+
+
+@pytest.mark.parametrize("text,lang", [
+    ("café", "pt"),
+    ("café", "en"),
+    ("café", "es"),
+    ("café", "arg"),
+    ("café", "her"),
+    ("café", "arn"),
+])
+def test_strip_raises_for_native_orthography_langs(text, lang):
+    with pytest.raises(ValueError):
+        strip_diacritics(text, lang)
+
+
+@pytest.mark.parametrize("lang", [
+    "ru", "uk", "be", "ar", "ar-SA", "he", "he-IL", "bg", "ka",
+])
+def test_supports_strip_true(lang):
+    assert _supports_strip(lang) is True
+
+
+@pytest.mark.parametrize("lang", [
+    "pt", "pt-PT", "en", "arg", "her", "arn",
+])
+def test_supports_strip_false(lang):
+    assert _supports_strip(lang) is False
+
+
+def test_overlay_marks_stress_lang():
+    lang = next(iter(STRESS_LANGS))
+    assert _overlay_marks(lang) == _STRESS_MARKS
+
+
+def test_overlay_marks_arabic():
+    assert _overlay_marks("ar") == _ARABIC_MARKS
+
+
+def test_overlay_marks_hebrew():
+    assert _overlay_marks("he") == _HEBREW_MARKS
+
+
+def test_overlay_marks_none_for_native_orthography():
+    assert _overlay_marks("pt") is None
+    assert _overlay_marks("en") is None

{scriptconv-0.0.4a4 → scriptconv-0.0.4a5}/tests/test_phonemizers_cjk_ar.py

@@ -21,6 +21,65 @@ class TestVendoredChinese(unittest.TestCase):
         from scriptconv.phonemizers._thirdparty.zh_num import num2str
         self.assertEqual(num2str("123"), "一百二十三")
 
+    def test_zh_num_fullwidth_digits_normalized(self):
+        # zh_num.DIGITS is keyed by ASCII "0"-"9"; full-width digits (e.g.
+        # "３") must be NFKC-folded before lookup or num2str raises KeyError
+        import unicodedata
+        from scriptconv.phonemizers._thirdparty.zh_num import num2str
+        fullwidth = "１２３"
+        self.assertEqual(
+            num2str(unicodedata.normalize("NFKC", fullwidth)),
+            num2str("123"),
+        )
+
+
+class TestPinyinRetone(unittest.TestCase):
+    def test_retone_tone_marks(self):
+        from scriptconv.phonemizers.zh import BaseChinesePinyinPhonemizer
+        retoned = BaseChinesePinyinPhonemizer._retone("ma˥")
+        self.assertEqual(retoned, "ma→")
+
+    def test_retone_leftover_syllabic_mark_does_not_raise(self):
+        # any pinyin_to_ipa output carrying an unanticipated combining
+        # U+0329 (not attached to ɻ/ɹ) must degrade gracefully, not crash
+        from scriptconv.phonemizers.zh import BaseChinesePinyinPhonemizer
+        leftover = "n" + chr(809)
+        retoned = BaseChinesePinyinPhonemizer._retone(leftover)
+        self.assertEqual(retoned, leftover)
+
+
+class TestZhBackendFriendlyImportErrors(unittest.TestCase):
+    def _assert_friendly(self, module_name, cls, extra):
+        # sys.modules[name] = None makes the `import` statement raise
+        # ImportError regardless of whether the package is actually
+        # installed, so this simulates the missing-dependency case cleanly
+        import sys
+        from unittest.mock import patch
+        with patch.dict(sys.modules, {module_name: None}):
+            with self.assertRaises(ImportError) as ctx:
+                cls()
+        self.assertIn(f"scriptconv[{extra}]", str(ctx.exception))
+
+    def test_pypinyin_missing_dep_message(self):
+        from scriptconv.phonemizers.zh import PypinyinPhonemizer
+        self._assert_friendly("pypinyin", PypinyinPhonemizer, "zh")
+
+    def test_xpinyin_missing_dep_message(self):
+        from scriptconv.phonemizers.zh import XpinyinPhonemizer
+        self._assert_friendly("xpinyin", XpinyinPhonemizer, "zh-phonemizers")
+
+    def test_g2pm_missing_dep_message(self):
+        from scriptconv.phonemizers.zh import G2pMPhonemizer
+        self._assert_friendly("g2pM", G2pMPhonemizer, "zh-phonemizers")
+
+    def test_g2pc_missing_dep_message(self):
+        from scriptconv.phonemizers.zh import G2pCPhonemizer
+        self._assert_friendly("g2pc", G2pCPhonemizer, "zh-phonemizers")
+
+    def test_pinyin_to_ipa_missing_dep_message(self):
+        from scriptconv.phonemizers.zh import PypinyinPhonemizer
+        self._assert_friendly("pinyin_to_ipa", PypinyinPhonemizer, "zh-phonemizers")
+
 
 class TestShamiFrontend(unittest.TestCase):
     def test_codeswitch_language_ids_align(self):

{scriptconv-0.0.4a4 → scriptconv-0.0.4a5}/tests/test_scripts.py

@@ -742,3 +742,15 @@ def test_script_runs_empty_and_pure_punctuation():
 def test_script_runs_reconstructs_input():
     for text in ["привет hello", "Hello مرحبا world", "приве́т", "  hi", "日本語 abc"]:
         assert "".join(t for _, t in script_runs(text)) == text
+
+
+def test_char_script_ranges_are_non_overlapping():
+    # char_script's binary search only checks one candidate interval; that
+    # is only correct if no two registered char_ranges overlap.
+    ranges = sorted(
+        (lo, hi)
+        for script in SCRIPT_REGISTRY.values()
+        for lo, hi in script.char_ranges
+    )
+    for (_, prev_hi), (next_lo, _) in zip(ranges, ranges[1:]):
+        assert next_lo > prev_hi