scriptconv 0.0.4a19__tar.gz → 0.0.4a20__tar.gz

{scriptconv-0.0.4a19/scriptconv.egg-info → scriptconv-0.0.4a20}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: scriptconv
-Version: 0.0.4a19
+Version: 0.0.4a20
 Summary: Zero-dependency script & phoneme-notation core — ISO-15924 detection & metadata, IPA↔ARPABET/X-SAMPA/Lexique/Kirshenbaum/Cotovía/RFE, Buckwalter↔Arabic, Hangul→jamo, kana
 License: Apache-2.0
 Project-URL: Homepage, https://github.com/TigreGotico/scriptconv
@@ -80,7 +80,7 @@ Requires-Dist: scriptconv[phonemizers]; extra == "tashkeel"
 Requires-Dist: text2tashkeel; extra == "tashkeel"
 Provides-Extra: stress
 Requires-Dist: scriptconv[phonemizers]; extra == "stress"
-Requires-Dist: stressonnx>=0.0.2; extra == "stress"
+Requires-Dist: stressonnx>=0.0.3a2; extra == "stress"
 Provides-Extra: pt
 Requires-Dist: scriptconv[phonemizers]; extra == "pt"
 Requires-Dist: bifonia>=0.1.1; extra == "pt"
@@ -123,6 +123,7 @@ Requires-Dist: langcodes; extra == "test"
 Requires-Dist: regex; extra == "test"
 Requires-Dist: ovos-number-parser; extra == "test"
 Requires-Dist: bifonia>=0.1.1; extra == "test"
+Requires-Dist: stressonnx>=0.0.3a2; extra == "test"
 Dynamic: license-file
 
 <div align="center">

{scriptconv-0.0.4a19 → scriptconv-0.0.4a20}/pyproject.toml

@@ -53,7 +53,7 @@ he = ["scriptconv[phonemizers]", "phonikud-onnx"]
 fa = ["scriptconv[phonemizers]", "persian_phonemizer"]
 vi = ["scriptconv[phonemizers]", "viphoneme"]
 tashkeel = ["scriptconv[phonemizers]", "text2tashkeel"]
-stress = ["scriptconv[phonemizers]", "stressonnx>=0.0.2"]
+stress = ["scriptconv[phonemizers]", "stressonnx>=0.0.3a2"]
 pt = ["scriptconv[phonemizers]", "bifonia>=0.1.1"]
 ja-phonemizers = ["scriptconv[phonemizers]", "pyopenjtalk", "cutlet", "pykakasi>=2.3,<3"]
 ko = ["scriptconv[phonemizers]", "g2pk", "regex"]
@@ -61,7 +61,7 @@ zh-phonemizers = ["scriptconv[phonemizers]", "jieba", "pypinyin>=0.50,<1", "xpin
 en-phonemizers = ["scriptconv[phonemizers]", "g2p_en", "openphonemizer", "deep-phonemizer", "requests"]
 ar-phonemizers = ["scriptconv[phonemizers]", "arbtok", "ovos-number-parser"]
 shami = ["scriptconv[phonemizers]", "regex"]
-test = ["pytest", "pytest-timeout", "pykakasi>=2.3,<3", "pypinyin>=0.50,<1", "quebra-frases", "langcodes", "regex", "ovos-number-parser", "bifonia>=0.1.1"]
+test = ["pytest", "pytest-timeout", "pykakasi>=2.3,<3", "pypinyin>=0.50,<1", "quebra-frases", "langcodes", "regex", "ovos-number-parser", "bifonia>=0.1.1", "stressonnx>=0.0.3a2"]
 
 [project.urls]
 Homepage = "https://github.com/TigreGotico/scriptconv"

{scriptconv-0.0.4a19 → scriptconv-0.0.4a20}/scriptconv/diacritics.py

@@ -94,17 +94,86 @@ def _is_european_portuguese(lang: str) -> bool:
     return norm == "pt" or norm == "pt-pt"
 
 
+#: Primary subtags whose STRESS_LANGS membership does not by itself mean the
+#: bare tag is servable — stressonnx only ships script-qualified variants
+#: (az-Cyrl/az-Latn, uz-Cyrl/uz-Latn) for these, so the bare tag is script-
+#: ambiguous. Used both by the real stressonnx-backed check and by the
+#: no-extra-installed fallback, so the two never disagree.
+_SCRIPT_AMBIGUOUS_STRESS_LANGS = frozenset({"az", "uz"})
+
+
+def _is_stress_lang_fallback(lang: str) -> bool:
+    """``_is_stress_lang`` semantics without importing stressonnx.
+
+    Mirrors stressonnx's own resolution rules closely enough to agree with
+    it on every tag ``STRESS_LANGS`` can produce: an exact/regional match on
+    a member whose primary subtag isn't script-ambiguous is accepted; a
+    4-letter script subtag is only meaningful for the script-ambiguous pair
+    (az/uz) and must be Cyrl/Latn; every other script-qualified tag, and
+    every bare az/uz, is rejected rather than guessed at.
+    """
+    if not isinstance(lang, str):
+        return False
+    normalized = lang.replace("_", "-")
+    parts = normalized.split("-")
+    p = parts[0].lower()
+    if p not in STRESS_LANGS:
+        return False
+    has_script = len(parts) >= 2 and len(parts[1]) == 4 and parts[1].isalpha()
+    if p in _SCRIPT_AMBIGUOUS_STRESS_LANGS:
+        return has_script and parts[1].title() in ("Cyrl", "Latn")
+    return not has_script
+
+
+def _is_stress_lang(lang: str) -> bool:
+    """True if stressonnx actually resolves *lang* to a supported model.
+
+    Delegates to stressonnx's own ``resolve_lang`` (the same resolver
+    :func:`_stress` triggers indirectly through ``stress()``) instead of
+    re-deriving the resolution rules here, so this and the real call can
+    never disagree — e.g. a bare ``"az"``/``"uz"`` is script-ambiguous and
+    stressonnx rejects it even though ``"az"``/``"uz"`` are primary subtags
+    of a ``STRESS_LANGS`` member, while a regional tag like ``"ru-RU"``
+    resolves fine to ``"ru"``. Only reached once :func:`_diacritizer_family`
+    has already matched the primary subtag against ``STRESS_LANGS``, so
+    non-stress tags (``en``, ``pt`` …) never pay for the stressonnx import
+    or the exception-based resolution.
+
+    Falls back to :func:`_is_stress_lang_fallback` — the same script-
+    ambiguity rule, evaluated without importing stressonnx — when the
+    optional ``[stress]`` extra isn't installed, so ``pip install
+    scriptconv[stress]`` changes only whether generation *works*, never
+    which tags are claimed as stress-family.
+    """
+    try:
+        from stressonnx import ALL_LANGS
+        from stressonnx.langs import resolve_lang
+        from stressonnx.errors import UnsupportedLanguageError
+    except ImportError:
+        return _is_stress_lang_fallback(lang)
+    try:
+        resolve_lang(lang, ALL_LANGS)
+    except UnsupportedLanguageError:
+        return False
+    return True
+
+
 def _diacritizer_family(lang: str) -> Optional[str]:
     """Which diacritization backend family handles *lang*, or ``None``.
 
-    Single source of truth for lang→backend routing: both the forward
-    dispatch (:func:`diacritize`) and the strip direction (:func:`_overlay_marks`)
-    resolve through this, so the two can never disagree about which language
-    uses which backend. Returns one of ``"he"`` (niqqud), ``"ar"`` (tashkeel),
-    ``"stress"`` (stressonnx), ``"pt"`` (bifonia sense diacritics), or
-    ``None``. Uses exact primary-subtag matching (never ``startswith``), so
-    Aragonese (``arg``), Herero (``her``) and Mapudungun (``arn``) are never
-    misread as Arabic/Hebrew.
+    Single source of truth for lang→backend routing for the *strip*
+    direction (:func:`_overlay_marks`) and for which languages carry
+    overlay marks at all: it uses the broad ``STRESS_LANGS`` primary-subtag
+    membership, not stressonnx's stricter script-resolution rules, because
+    stripping needs no model and every ``STRESS_LANGS`` member (bare az/uz
+    included) writes its overlay marks the same way regardless of script
+    ambiguity. :func:`diacritize`'s *generation* path narrows the "stress"
+    result further through :func:`_is_stress_lang` before actually calling
+    stressonnx, since only that path needs the real resolution. Returns one
+    of ``"he"`` (niqqud), ``"ar"`` (tashkeel), ``"stress"`` (stressonnx),
+    ``"pt"`` (bifonia sense diacritics), or ``None``. Uses exact primary-
+    subtag matching (never ``startswith``), so Aragonese (``arg``), Herero
+    (``her``) and Mapudungun (``arn``) are never misread as Arabic/Hebrew.
     """
     p = _primary_subtag(lang)
     if p == "he":
@@ -285,7 +354,14 @@ def diacritize(text: str, lang: str = "und", diacritizer_model=None,
     if family == "ar":
         return _tashkeel(diacritizer_model).diacritize(text)
     if family == "stress":
-        return _stress(text, lang, diacritizer_model)
+        # _diacritizer_family's "stress" match is the broad STRESS_LANGS
+        # primary-subtag test (also used for stripping); generation needs
+        # stressonnx's stricter, script-aware resolution — a bare az/uz or
+        # a script-mismatched regional tag (e.g. "ru-Latn") is script-
+        # ambiguous/unsupported and must not reach the backend at all.
+        if _is_stress_lang(lang):
+            return _stress(text, lang, diacritizer_model)
+        return text
     if family == "pt":
         return _sense_diacritics_pt(text)
     return text

{scriptconv-0.0.4a19 → scriptconv-0.0.4a20}/scriptconv/version.py

@@ -2,7 +2,7 @@
 VERSION_MAJOR = 0
 VERSION_MINOR = 0
 VERSION_BUILD = 4
-VERSION_ALPHA = 19
+VERSION_ALPHA = 20
 # END_VERSION_BLOCK
 
 VERSION_STR = f"{VERSION_MAJOR}.{VERSION_MINOR}.{VERSION_BUILD}"

{scriptconv-0.0.4a19 → scriptconv-0.0.4a20/scriptconv.egg-info}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: scriptconv
-Version: 0.0.4a19
+Version: 0.0.4a20
 Summary: Zero-dependency script & phoneme-notation core — ISO-15924 detection & metadata, IPA↔ARPABET/X-SAMPA/Lexique/Kirshenbaum/Cotovía/RFE, Buckwalter↔Arabic, Hangul→jamo, kana
 License: Apache-2.0
 Project-URL: Homepage, https://github.com/TigreGotico/scriptconv
@@ -80,7 +80,7 @@ Requires-Dist: scriptconv[phonemizers]; extra == "tashkeel"
 Requires-Dist: text2tashkeel; extra == "tashkeel"
 Provides-Extra: stress
 Requires-Dist: scriptconv[phonemizers]; extra == "stress"
-Requires-Dist: stressonnx>=0.0.2; extra == "stress"
+Requires-Dist: stressonnx>=0.0.3a2; extra == "stress"
 Provides-Extra: pt
 Requires-Dist: scriptconv[phonemizers]; extra == "pt"
 Requires-Dist: bifonia>=0.1.1; extra == "pt"
@@ -123,6 +123,7 @@ Requires-Dist: langcodes; extra == "test"
 Requires-Dist: regex; extra == "test"
 Requires-Dist: ovos-number-parser; extra == "test"
 Requires-Dist: bifonia>=0.1.1; extra == "test"
+Requires-Dist: stressonnx>=0.0.3a2; extra == "test"
 Dynamic: license-file
 
 <div align="center">

{scriptconv-0.0.4a19 → scriptconv-0.0.4a20}/scriptconv.egg-info/requires.txt

@@ -94,7 +94,7 @@ regex
 
 [stress]
 scriptconv[phonemizers]
-stressonnx>=0.0.2
+stressonnx>=0.0.3a2
 
 [tashkeel]
 scriptconv[phonemizers]
@@ -110,6 +110,7 @@ langcodes
 regex
 ovos-number-parser
 bifonia>=0.1.1
+stressonnx>=0.0.3a2
 
 [transphone]
 scriptconv[phonemizers]

{scriptconv-0.0.4a19 → scriptconv-0.0.4a20}/tests/test_diacritics.py

@@ -47,6 +47,15 @@ def test_strip_hebrew_removes_niqqud_preserves_consonants():
     assert stripped == "שלום"
 
 
+def test_strip_bare_az_uses_broad_stress_lang_membership():
+    # Bare "az"/"uz" are script-ambiguous for *generation* (stressonnx needs
+    # az-Cyrl/az-Latn to actually run), but stripping is a pure codepoint
+    # filter that needs no model — it must still work for the bare tag
+    # rather than raising the native-orthography ValueError.
+    text = "za" + "\u0301" + "mok"  # combining acute (U+0301), not precomposed á
+    assert strip_diacritics(text, "az") == "zamok"
+
+
 @pytest.mark.parametrize("text,lang", [
     ("café", "pt"),
     ("café", "en"),
@@ -75,7 +84,9 @@ def test_supports_strip_false(lang):
 
 
 def test_overlay_marks_stress_lang():
-    lang = next(iter(STRESS_LANGS))
+    # "az"/"uz" are script-ambiguous bare tags (stressonnx only serves the
+    # az-Cyrl/az-Latn/uz-Cyrl/uz-Latn variants) — pick an unambiguous member.
+    lang = next(t for t in STRESS_LANGS if t not in ("az", "uz"))
     assert _overlay_marks(lang) == _STRESS_MARKS
 
 
@@ -294,3 +305,108 @@ class TestEastSlavicStressRouting:
             with pytest.raises(ImportError) as ctx:
                 diacritize("замок", "ru")
         assert "stressonnx" in str(ctx.value)
+
+
+class TestFamilyAgreesWithStressonnxResolution:
+    """``_diacritizer_family`` must never claim a lang stressonnx rejects,
+    and must never reject a lang stressonnx actually resolves — the two
+    have to agree by construction (real installed stressonnx 0.0.3a2, no
+    model inference: ``_stress`` itself is monkeypatched)."""
+
+    def test_regional_tag_routes_to_stress_and_calls_through_with_raw_tag(self, monkeypatch):
+        from scriptconv import diacritics
+        calls = []
+        monkeypatch.setattr(diacritics, "_stress",
+                             lambda text, lang, model=None: calls.append((text, lang)) or "STRESSED")
+        out = diacritics.diacritize("замок", "ru-RU")
+        assert out == "STRESSED"
+        assert calls == [("замок", "ru-RU")]
+
+    def test_bare_az_does_not_route_to_stress(self, monkeypatch):
+        from scriptconv import diacritics
+        monkeypatch.setattr(diacritics, "_stress",
+                             lambda text, lang, model=None: (_ for _ in ()).throw(
+                                 AssertionError("stress backend must not be called for bare 'az'")))
+        # bare "az" is script-ambiguous for stressonnx (only az-Cyrl/az-Latn
+        # exist) — no UnsupportedLanguageError should escape scriptconv, and
+        # the stress backend must never be invoked for it.
+        out = diacritics.diacritize("salam", "az")
+        assert out == "salam"
+
+    def test_bare_uz_does_not_route_to_stress(self, monkeypatch):
+        from scriptconv import diacritics
+        monkeypatch.setattr(diacritics, "_stress",
+                             lambda text, lang, model=None: (_ for _ in ()).throw(
+                                 AssertionError("stress backend must not be called for bare 'uz'")))
+        out = diacritics.diacritize("salom", "uz")
+        assert out == "salom"
+
+    def test_script_qualified_az_routes_to_stress(self, monkeypatch):
+        from scriptconv import diacritics
+        calls = []
+        monkeypatch.setattr(diacritics, "_stress",
+                             lambda text, lang, model=None: calls.append((text, lang)) or "STRESSED")
+        out = diacritics.diacritize("salam", "az-Latn")
+        assert out == "STRESSED"
+        assert calls == [("salam", "az-Latn")]
+
+    def test_ru_latn_does_not_crash_and_is_not_claimed_by_stress(self, monkeypatch):
+        from scriptconv import diacritics
+        monkeypatch.setattr(diacritics, "_stress",
+                             lambda text, lang, model=None: (_ for _ in ()).throw(
+                                 AssertionError("stress backend must not be called for 'ru-Latn'")))
+        # "ru-Latn" names a script Russian isn't written in — stressonnx
+        # raises UnsupportedLanguageError for it, so the family test must
+        # not claim it either; scriptconv falls through to unchanged text.
+        text = "zamok"
+        assert diacritics.diacritize(text, "ru-Latn") == text
+
+    def test_fallback_matches_real_stressonnx_on_every_probe(self):
+        # ``pip install scriptconv[stress]`` must change only whether
+        # generation *works*, never which tags are claimed as stress-family
+        # — so the no-extra-installed fallback and the real, installed
+        # stressonnx must agree on every probe tag.
+        from scriptconv.diacritics import _is_stress_lang, _is_stress_lang_fallback
+
+        probes = ["ru", "ru-RU", "ru-Latn", "az", "uz", "az-Latn", "az-Cyrl",
+                  "uz-Latn", "uz-Cyrl", "uk", "be", "ka", "hy", "ber", "en"]
+        for tag in probes:
+            assert _is_stress_lang_fallback(tag) == _is_stress_lang(tag), tag
+
+
+class TestStressFamilyRoutingWithoutStressonnxInstalled:
+    """``_is_stress_lang`` must apply the same script-ambiguity rule whether
+    or not stressonnx is importable, so routing never depends on the
+    optional extra being installed."""
+
+    def _hide_stressonnx(self, monkeypatch):
+        import sys
+        monkeypatch.setitem(sys.modules, "stressonnx", None)
+        monkeypatch.setitem(sys.modules, "stressonnx.langs", None)
+        monkeypatch.setitem(sys.modules, "stressonnx.errors", None)
+
+    def test_bare_az_not_claimed_without_extra(self, monkeypatch):
+        from scriptconv import diacritics
+        self._hide_stressonnx(monkeypatch)
+        assert diacritics._is_stress_lang("az") is False
+        assert diacritics.diacritize("salam", "az") == "salam"
+
+    def test_bare_uz_not_claimed_without_extra(self, monkeypatch):
+        from scriptconv import diacritics
+        self._hide_stressonnx(monkeypatch)
+        assert diacritics._is_stress_lang("uz") is False
+
+    def test_script_qualified_az_claimed_without_extra(self, monkeypatch):
+        from scriptconv import diacritics
+        self._hide_stressonnx(monkeypatch)
+        assert diacritics._is_stress_lang("az-Latn") is True
+
+    def test_ru_latn_not_claimed_without_extra(self, monkeypatch):
+        from scriptconv import diacritics
+        self._hide_stressonnx(monkeypatch)
+        assert diacritics._is_stress_lang("ru-Latn") is False
+
+    def test_regional_ru_claimed_without_extra(self, monkeypatch):
+        from scriptconv import diacritics
+        self._hide_stressonnx(monkeypatch)
+        assert diacritics._is_stress_lang("ru-RU") is True