scriptconv 0.0.4a18__tar.gz → 0.0.4a20__tar.gz

{scriptconv-0.0.4a18/scriptconv.egg-info → scriptconv-0.0.4a20}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: scriptconv
-Version: 0.0.4a18
+Version: 0.0.4a20
 Summary: Zero-dependency script & phoneme-notation core — ISO-15924 detection & metadata, IPA↔ARPABET/X-SAMPA/Lexique/Kirshenbaum/Cotovía/RFE, Buckwalter↔Arabic, Hangul→jamo, kana
 License: Apache-2.0
 Project-URL: Homepage, https://github.com/TigreGotico/scriptconv
@@ -80,7 +80,7 @@ Requires-Dist: scriptconv[phonemizers]; extra == "tashkeel"
 Requires-Dist: text2tashkeel; extra == "tashkeel"
 Provides-Extra: stress
 Requires-Dist: scriptconv[phonemizers]; extra == "stress"
-Requires-Dist: stressonnx>=0.0.2; extra == "stress"
+Requires-Dist: stressonnx>=0.0.3a2; extra == "stress"
 Provides-Extra: pt
 Requires-Dist: scriptconv[phonemizers]; extra == "pt"
 Requires-Dist: bifonia>=0.1.1; extra == "pt"
@@ -123,6 +123,7 @@ Requires-Dist: langcodes; extra == "test"
 Requires-Dist: regex; extra == "test"
 Requires-Dist: ovos-number-parser; extra == "test"
 Requires-Dist: bifonia>=0.1.1; extra == "test"
+Requires-Dist: stressonnx>=0.0.3a2; extra == "test"
 Dynamic: license-file
 
 <div align="center">

{scriptconv-0.0.4a18 → scriptconv-0.0.4a20}/pyproject.toml

@@ -53,7 +53,7 @@ he = ["scriptconv[phonemizers]", "phonikud-onnx"]
 fa = ["scriptconv[phonemizers]", "persian_phonemizer"]
 vi = ["scriptconv[phonemizers]", "viphoneme"]
 tashkeel = ["scriptconv[phonemizers]", "text2tashkeel"]
-stress = ["scriptconv[phonemizers]", "stressonnx>=0.0.2"]
+stress = ["scriptconv[phonemizers]", "stressonnx>=0.0.3a2"]
 pt = ["scriptconv[phonemizers]", "bifonia>=0.1.1"]
 ja-phonemizers = ["scriptconv[phonemizers]", "pyopenjtalk", "cutlet", "pykakasi>=2.3,<3"]
 ko = ["scriptconv[phonemizers]", "g2pk", "regex"]
@@ -61,7 +61,7 @@ zh-phonemizers = ["scriptconv[phonemizers]", "jieba", "pypinyin>=0.50,<1", "xpin
 en-phonemizers = ["scriptconv[phonemizers]", "g2p_en", "openphonemizer", "deep-phonemizer", "requests"]
 ar-phonemizers = ["scriptconv[phonemizers]", "arbtok", "ovos-number-parser"]
 shami = ["scriptconv[phonemizers]", "regex"]
-test = ["pytest", "pytest-timeout", "pykakasi>=2.3,<3", "pypinyin>=0.50,<1", "quebra-frases", "langcodes", "regex", "ovos-number-parser", "bifonia>=0.1.1"]
+test = ["pytest", "pytest-timeout", "pykakasi>=2.3,<3", "pypinyin>=0.50,<1", "quebra-frases", "langcodes", "regex", "ovos-number-parser", "bifonia>=0.1.1", "stressonnx>=0.0.3a2"]
 
 [project.urls]
 Homepage = "https://github.com/TigreGotico/scriptconv"
@@ -76,6 +76,7 @@ scriptconv = ["py.typed"]
 "scriptconv.phonemizers._thirdparty" = ["ko_tables/*.csv", "ko_tables/*.tsv"]
 "scriptconv.phonemizers._vendored.kog2p" = ["rulebook.txt", "LICENSE.md"]
 "scriptconv.phonemizers._vendored.mantoq" = ["LICENSE.md"]
+"scriptconv.phonemizers._vendored.iqra_phonetiser" = ["LICENSE.md"]
 "scriptconv.phonemizers._vendored.africa_g2p" = [
     "LICENSE.md", "DATA_LICENSE.md",
     "languages/*.json", "data/*.json",

{scriptconv-0.0.4a18 → scriptconv-0.0.4a20}/scriptconv/diacritics.py

@@ -94,17 +94,86 @@ def _is_european_portuguese(lang: str) -> bool:
     return norm == "pt" or norm == "pt-pt"
 
 
+#: Primary subtags whose STRESS_LANGS membership does not by itself mean the
+#: bare tag is servable — stressonnx only ships script-qualified variants
+#: (az-Cyrl/az-Latn, uz-Cyrl/uz-Latn) for these, so the bare tag is script-
+#: ambiguous. Used both by the real stressonnx-backed check and by the
+#: no-extra-installed fallback, so the two never disagree.
+_SCRIPT_AMBIGUOUS_STRESS_LANGS = frozenset({"az", "uz"})
+
+
+def _is_stress_lang_fallback(lang: str) -> bool:
+    """``_is_stress_lang`` semantics without importing stressonnx.
+
+    Mirrors stressonnx's own resolution rules closely enough to agree with
+    it on every tag ``STRESS_LANGS`` can produce: an exact/regional match on
+    a member whose primary subtag isn't script-ambiguous is accepted; a
+    4-letter script subtag is only meaningful for the script-ambiguous pair
+    (az/uz) and must be Cyrl/Latn; every other script-qualified tag, and
+    every bare az/uz, is rejected rather than guessed at.
+    """
+    if not isinstance(lang, str):
+        return False
+    normalized = lang.replace("_", "-")
+    parts = normalized.split("-")
+    p = parts[0].lower()
+    if p not in STRESS_LANGS:
+        return False
+    has_script = len(parts) >= 2 and len(parts[1]) == 4 and parts[1].isalpha()
+    if p in _SCRIPT_AMBIGUOUS_STRESS_LANGS:
+        return has_script and parts[1].title() in ("Cyrl", "Latn")
+    return not has_script
+
+
+def _is_stress_lang(lang: str) -> bool:
+    """True if stressonnx actually resolves *lang* to a supported model.
+
+    Delegates to stressonnx's own ``resolve_lang`` (the same resolver
+    :func:`_stress` triggers indirectly through ``stress()``) instead of
+    re-deriving the resolution rules here, so this and the real call can
+    never disagree — e.g. a bare ``"az"``/``"uz"`` is script-ambiguous and
+    stressonnx rejects it even though ``"az"``/``"uz"`` are primary subtags
+    of a ``STRESS_LANGS`` member, while a regional tag like ``"ru-RU"``
+    resolves fine to ``"ru"``. Only reached once :func:`_diacritizer_family`
+    has already matched the primary subtag against ``STRESS_LANGS``, so
+    non-stress tags (``en``, ``pt`` …) never pay for the stressonnx import
+    or the exception-based resolution.
+
+    Falls back to :func:`_is_stress_lang_fallback` — the same script-
+    ambiguity rule, evaluated without importing stressonnx — when the
+    optional ``[stress]`` extra isn't installed, so ``pip install
+    scriptconv[stress]`` changes only whether generation *works*, never
+    which tags are claimed as stress-family.
+    """
+    try:
+        from stressonnx import ALL_LANGS
+        from stressonnx.langs import resolve_lang
+        from stressonnx.errors import UnsupportedLanguageError
+    except ImportError:
+        return _is_stress_lang_fallback(lang)
+    try:
+        resolve_lang(lang, ALL_LANGS)
+    except UnsupportedLanguageError:
+        return False
+    return True
+
+
 def _diacritizer_family(lang: str) -> Optional[str]:
     """Which diacritization backend family handles *lang*, or ``None``.
 
-    Single source of truth for lang→backend routing: both the forward
-    dispatch (:func:`diacritize`) and the strip direction (:func:`_overlay_marks`)
-    resolve through this, so the two can never disagree about which language
-    uses which backend. Returns one of ``"he"`` (niqqud), ``"ar"`` (tashkeel),
-    ``"stress"`` (stressonnx), ``"pt"`` (bifonia sense diacritics), or
-    ``None``. Uses exact primary-subtag matching (never ``startswith``), so
-    Aragonese (``arg``), Herero (``her``) and Mapudungun (``arn``) are never
-    misread as Arabic/Hebrew.
+    Single source of truth for lang→backend routing for the *strip*
+    direction (:func:`_overlay_marks`) and for which languages carry
+    overlay marks at all: it uses the broad ``STRESS_LANGS`` primary-subtag
+    membership, not stressonnx's stricter script-resolution rules, because
+    stripping needs no model and every ``STRESS_LANGS`` member (bare az/uz
+    included) writes its overlay marks the same way regardless of script
+    ambiguity. :func:`diacritize`'s *generation* path narrows the "stress"
+    result further through :func:`_is_stress_lang` before actually calling
+    stressonnx, since only that path needs the real resolution. Returns one
+    of ``"he"`` (niqqud), ``"ar"`` (tashkeel), ``"stress"`` (stressonnx),
+    ``"pt"`` (bifonia sense diacritics), or ``None``. Uses exact primary-
+    subtag matching (never ``startswith``), so Aragonese (``arg``), Herero
+    (``her``) and Mapudungun (``arn``) are never misread as Arabic/Hebrew.
     """
     p = _primary_subtag(lang)
     if p == "he":
@@ -285,7 +354,14 @@ def diacritize(text: str, lang: str = "und", diacritizer_model=None,
     if family == "ar":
         return _tashkeel(diacritizer_model).diacritize(text)
     if family == "stress":
-        return _stress(text, lang, diacritizer_model)
+        # _diacritizer_family's "stress" match is the broad STRESS_LANGS
+        # primary-subtag test (also used for stripping); generation needs
+        # stressonnx's stricter, script-aware resolution — a bare az/uz or
+        # a script-mismatched regional tag (e.g. "ru-Latn") is script-
+        # ambiguous/unsupported and must not reach the backend at all.
+        if _is_stress_lang(lang):
+            return _stress(text, lang, diacritizer_model)
+        return text
     if family == "pt":
         return _sense_diacritics_pt(text)
     return text

{scriptconv-0.0.4a18 → scriptconv-0.0.4a20}/scriptconv/notation.py

@@ -42,6 +42,7 @@ __all__ = [
     "rfe_to_ipa",
     "ipa_to_rfe",
     "halabi_to_ipa",
+    "iqra_halabi_to_ipa",
     "looks_like_ipa",
 ]
 
@@ -1048,10 +1049,20 @@ _HALABI_VOWELS: dict[str, str] = {
     "a": "a", "aa": "aː", "aaaa": "aːː",
     "i": "i", "ii": "iː",
     "u": "u", "uu": "uː",
+    # emphatic-context allophones from the RAW (pre-mantoq-simplification)
+    # Halabi phonetiser layer — mantoq's own g2p folds these to the plain
+    # forms above via ``simplify_phonemes``, but scriptconv's raw
+    # ``Phonemizer.HALABI``/``Phonemizer.IQRA`` edges call the phonetiser
+    # directly and preserve the distinction (see arbtok's
+    # scripts/benchmark_iqraeval.py, verified symbol-by-symbol against the
+    # IqraEval/Iqra_train corpus).
+    "A": "ɑ", "AA": "ɑː",
+    "I": "ɪ", "II": "ɪː",
+    "U": "ʊ", "UU": "ʊː",
 }
 
 _HALABI_MULTI = ("_dbl_", "_+_", "_sil_", "_eos_", "_pad_",
-                 "aaaa", "aa", "ii", "uu")
+                 "aaaa", "aa", "ii", "uu", "AA", "II", "UU")
 
 
 def _tokenize_halabi(text: str) -> list[str]:
@@ -1136,6 +1147,67 @@ def halabi_to_ipa(halabi: str | list[str], errors: str = "pass") -> str:
     return "".join(out)
 
 
+# ---------------------------------------------------------------------------
+# IqraEval phoneme_ref -> IPA
+#
+# The IqraEval shared task (Interspeech 2025, doi:10.21437/Interspeech.2025-2411
+# — "we employed the phonetizer introduced by Nawar Halabi") scores against a
+# space-separated ``phoneme_ref`` column that is the raw Halabi phonetiser
+# output with the stress/vowel-realization digits, the ``+`` word separator
+# and the ``sil`` pause token stripped — but, unlike mantoq's own
+# ``simplify_phonemes``, keeping the emphatic-allophone uppercase vowels
+# (``A``/``AA``/``I``/``II``/``U``/``UU``) and gemination as a literal doubled
+# consonant letter (``bb``, ``nn``, …) rather than a ``_dbl_`` marker. Table
+# verified symbol-by-symbol against a 5,050-row IqraEval/Iqra_train sample by
+# TigreGotico/arbtok's ``scripts/benchmark_iqraeval.py``; reproduced here
+# (same license posture as ``halabi_to_ipa`` above: a factual symbol table,
+# not the CC BY-NC phonetiser code).
+# ---------------------------------------------------------------------------
+
+_IQRA_CONSONANTS: dict[str, str] = {
+    "b": "b", "t": "t", "^": "θ", "j": "dʒ", "H": "ħ", "x": "x", "d": "d",
+    "*": "ð", "r": "r", "z": "z", "s": "s", "$": "ʃ", "S": "sˤ", "D": "dˤ",
+    "T": "tˤ", "Z": "ðˤ", "E": "ʕ", "g": "ɣ", "f": "f", "q": "q", "k": "k",
+    "l": "l", "m": "m", "n": "n", "h": "h", "w": "w", "y": "j", "<": "ʔ",
+}
+_IQRA_VOWELS: dict[str, str] = {
+    "a": "a", "aa": "aː", "A": "ɑ", "AA": "ɑː",
+    "i": "i", "ii": "iː", "I": "ɪ", "II": "ɪː",
+    "u": "u", "uu": "uː", "U": "ʊ", "UU": "ʊː",
+}
+_IQRA_NOTATION: dict[str, str] = dict(_IQRA_VOWELS)
+for _sym, _ipa in _IQRA_CONSONANTS.items():
+    _IQRA_NOTATION[_sym] = _ipa
+    _IQRA_NOTATION[_sym * 2] = _ipa + "ː"  # gemination: doubled token, shadda
+
+
+def iqra_halabi_to_ipa(tokens: str, errors: str = "pass") -> str:
+    """Convert IqraEval-flavored (already digit/separator/sil-stripped)
+    space-separated Halabi tokens to IPA.
+
+    Unlike :func:`halabi_to_ipa` (which targets mantoq's own simplified,
+    lowercase-only inventory), this keeps the emphatic-allophone uppercase
+    vowels distinct and reads gemination off a literal doubled consonant
+    letter — the ``phoneme_ref`` convention.  Unknown tokens follow the
+    ``errors`` policy (``"pass"`` by default: passed through verbatim).
+
+    Examples
+    --------
+    >>> iqra_halabi_to_ipa("f ii h i x A y r aa t u n")
+    'fiːhixɑjraːtun'
+    """
+    out = []
+    for pos, tok in enumerate(tokens.split()):
+        ipa = _IQRA_NOTATION.get(tok)
+        if ipa is not None:
+            out.append(ipa)
+        else:
+            res = _unknown(tok, pos, "halabi", errors, replacement=f"{{{{{tok}}}}}")
+            if res:
+                out.append(res)
+    return "".join(out)
+
+
 # ---------------------------------------------------------------------------
 # convert — facade routing through IPA where no direct map exists
 # ---------------------------------------------------------------------------

scriptconv-0.0.4a20/scriptconv/phonemizers/_vendored/iqra_phonetiser/LICENSE.md

@@ -0,0 +1,28 @@
+# Third-party license notice — iqra_phonetiser
+
+This directory vendors the phonetisation core of `phonetise_Arabic.py` from
+[Iqra-Eval/MSA_phonetiser](https://github.com/Iqra-Eval/MSA_phonetiser) — the
+IqraEval shared task's own published data-prep code (used, per its
+`run_phonetiser.py`, to generate the `phoneme_ref` column of
+`IqraEval/Iqra_train`, the corpus `Phonemizer.IQRA` targets).
+
+That file is itself a fork of Nawar Halabi's Arabic-Phonetiser
+(https://github.com/nawarhalabi/Arabic-Phonetiser), licensed under the
+**Creative Commons Attribution-NonCommercial 4.0 International License**
+(https://creativecommons.org/licenses/by-nc/4.0/) — NOT the Apache-2.0 license
+of the rest of this repository. `phonetise-Buckwalter.py` in the IqraEval fork
+is byte-identical to the original upstream file (`cmp` verified);
+`phonetise_Arabic.py` carries small, deliberate patches on top of it (see
+`phonetiser.py`'s module docstring for the diff against both the original
+upstream and against mantoq's own vendored copy). The IqraEval fork carries no
+separate LICENSE file of its own, so the inherited CC BY-NC 4.0 terms govern
+it as a derivative work.
+
+Kept in its own quarantine, separate from `../mantoq/` — the two are
+independent copies of the same phonetiser family at different patch levels,
+each with the specific behavior a different scriptconv edge needs preserved:
+`../mantoq/` for `Phonemizer.MANTOQ`'s long-standing, model-compatible
+contract; this one for `Phonemizer.HALABI`/`Phonemizer.IQRA`'s fidelity to
+`phoneme_ref`. Nothing here imports at package import time; using it is an
+explicit per-request opt-in (constructing `HalabiPhonemizer` or
+`IqraPhonemizer`) that accepts this directory's license.

scriptconv-0.0.4a20/scriptconv/phonemizers/_vendored/iqra_phonetiser/__init__.py

@@ -0,0 +1,7 @@
+"""Quarantined vendored copy — see ``LICENSE.md`` in this directory.
+
+Nothing imports at package import time; callers reach in explicitly via
+``scriptconv.phonemizers._vendored.iqra_phonetiser.phonetiser`` (what
+:class:`scriptconv.phonemizers.ar.IqraPhonemizer` does), matching the same
+opt-in pattern as ``_vendored/mantoq``.
+"""