scriptconv 0.0.4a3__tar.gz → 0.0.4a4__tar.gz

{scriptconv-0.0.4a3/scriptconv.egg-info → scriptconv-0.0.4a4}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: scriptconv
-Version: 0.0.4a3
+Version: 0.0.4a4
 Summary: Zero-dependency script & phoneme-notation core — ISO-15924 detection & metadata, IPA↔ARPABET/X-SAMPA/Lexique/Kirshenbaum/Cotovía/RFE, Buckwalter↔Arabic, Hangul→jamo, kana
 License: Apache-2.0
 Project-URL: Homepage, https://github.com/TigreGotico/scriptconv
@@ -391,6 +391,17 @@ p = GraphemePhonemizer()
 p.add_diacritics("Tenho muita sede hoje.", "pt")   # 'Tenho muita sêde hoje.' (thirst)
 ```
 
+Diacritization also joins the graph, like phonemization, via
+`scriptconv.diacritics.register` — a `text -> text-diacritized` edge, opt-in,
+with `text -> ipa` routing unchanged:
+
+```python
+from scriptconv import diacritics
+g = DEFAULT_GRAPH.extend(diacritics.register).extend(phonemizers.register)
+g.convert("Tenho muita sede hoje.", "text", "text-diacritized", lang="pt")
+# 'Tenho muita sêde hoje.'
+```
+
 Stress is unwritten or under-marked in all 26 covered languages; East Slavic
 is the clearest case, where unstressed vowels also reduce (e.g. Russian
 о→[ɐ]/[ə]), so a missing mark corrupts more than prosody there. stressonnx is

{scriptconv-0.0.4a3 → scriptconv-0.0.4a4}/README.md

@@ -265,6 +265,17 @@ p = GraphemePhonemizer()
 p.add_diacritics("Tenho muita sede hoje.", "pt")   # 'Tenho muita sêde hoje.' (thirst)
 ```
 
+Diacritization also joins the graph, like phonemization, via
+`scriptconv.diacritics.register` — a `text -> text-diacritized` edge, opt-in,
+with `text -> ipa` routing unchanged:
+
+```python
+from scriptconv import diacritics
+g = DEFAULT_GRAPH.extend(diacritics.register).extend(phonemizers.register)
+g.convert("Tenho muita sede hoje.", "text", "text-diacritized", lang="pt")
+# 'Tenho muita sêde hoje.'
+```
+
 Stress is unwritten or under-marked in all 26 covered languages; East Slavic
 is the clearest case, where unstressed vowels also reduce (e.g. Russian
 о→[ɐ]/[ə]), so a missing mark corrupts more than prosody there. stressonnx is

scriptconv-0.0.4a4/scriptconv/diacritics.py

@@ -0,0 +1,153 @@
+"""Diacritization as a graph extension — parallel to :mod:`scriptconv.phonemizers`.
+
+Diacritization (Arabic tashkeel, Hebrew niqqud, East-Slavic/Turkic/Caucasian
+word stress, European-Portuguese homograph sense marks) is, architecturally,
+just another transform between text representations: it maps the ``"text"``
+node to a lang-contextual ``"text-diacritized"`` node.  Like
+:func:`scriptconv.phonemizers.register` it is opt-in — ``DEFAULT_GRAPH`` stays
+free of it until a caller extends a graph::
+
+    from scriptconv.graph import DEFAULT_GRAPH
+    from scriptconv import diacritics, phonemizers
+    g = DEFAULT_GRAPH.extend(diacritics.register).extend(phonemizers.register)
+
+    g.convert("замок стоит на горе", "text", "text-diacritized", lang="ru")
+    # 'за́мок сто́ит на горе́'    — just the diacritized text
+
+    g.convert("Tenho muita sede hoje.", "text", "text-diacritized", lang="pt")
+    # 'Tenho muita sêde hoje.'
+
+Routing ``"text" -> "ipa"`` still takes the direct phonemization edge by
+default: the diacritization edge is model-based (``lossless=False``), so it
+never out-prices a direct phonemization, and enabling this extension does not
+silently change phonemization output.  The diacritized route is taken only
+when a caller asks for ``"text-diacritized"`` explicitly.
+
+A future per-engine *stance* would push this further into topology rather than
+a flag: an engine that *requires* vocalized input would carry only a
+``"text-diacritized" -> "ipa"`` edge (forcing the detour), one that *forbids*
+diacritics would carry only ``"text" -> "ipa"`` (making the detour
+unroutable), and a tolerant engine (e.g. arbtok, which self-vocalizes) carries
+both.  The strip direction (``"text-diacritized" -> "text"``) is registered too, but
+gated to languages whose marks are removable *overlay* diacritics —
+Arabic/Hebrew vocalization and East-Slavic/Turkic/Caucasian stress. Strip
+removes only the specific overlay codepoints each backend adds (combining
+acute/grave for stress, tashkeel for Arabic, niqqud for Hebrew) — never a
+blanket combining-mark filter — so precomposed native letters survive
+(Cyrillic й/ё, Latvian macrons, Azerbaijani ç/ö, Arabic hamza carriers أ إ آ ؤ
+ئ). The gate uses exact primary-subtag matching, so Aragonese (``arg``),
+Herero (``her``), Mapudungun (``arn``) etc. are never misread as Arabic/Hebrew.
+Languages whose diacritics are native orthography (European Portuguese, via
+bifonia) refuse the strip with :class:`ValueError` instead of silently
+corrupting the spelling (``café`` must never become ``cafe``). This split
+tracks which backend/model produced the marks: overlay backends (phonikud,
+tashkeel, stressonnx) are strippable; spelling-integral backends (bifonia) are
+not.
+"""
+from scriptconv.graph import Edge
+from scriptconv.phonemizers.base import STRESS_LANGS, _primary_subtag
+
+#: The lang-contextual node produced by diacritization.  Like ``"text"`` it is
+#: meaningful only with ``lang=`` context and exists only in opted-in graphs.
+DIACRITIZED = "text-diacritized"
+
+_DEFAULT_PHONEMIZER = None
+
+
+def _get_phonemizer(phonikud_model=None):
+    global _DEFAULT_PHONEMIZER
+    if phonikud_model:
+        from scriptconv.phonemizers.base import GraphemePhonemizer
+        return GraphemePhonemizer(phonikud_model=phonikud_model)
+    if _DEFAULT_PHONEMIZER is None:
+        from scriptconv.phonemizers.base import GraphemePhonemizer
+        _DEFAULT_PHONEMIZER = GraphemePhonemizer()
+    return _DEFAULT_PHONEMIZER
+
+
+def diacritize(text: str, lang: str = "und", model=None, **context) -> str:
+    """Add pronunciation-disambiguating diacritics to *text* for *lang*.
+
+    Thin wrapper over :meth:`BasePhonemizer.add_diacritics` — the single
+    dispatch (he→phonikud, ar→tashkeel, ru/uk/be/…→stressonnx, pt→bifonia).
+    Reuses a cached default phonemizer so repeated calls don't reload models; a
+    per-call Hebrew ``phonikud_model`` path (via context) builds a fresh one.
+    """
+    return _get_phonemizer(context.get("phonikud_model")).add_diacritics(text, lang, model)
+
+
+# Combining marks each diacritization backend overlays onto the bare text.
+# Strip removes ONLY these codepoints — never a blanket combining-mark filter,
+# which (via NFD) would decompose and destroy precomposed native letters:
+# Cyrillic й/ё, Latvian ī, Azerbaijani ç/ö, and Arabic hamza carriers أ إ آ ؤ ئ.
+_STRESS_MARKS = frozenset({0x0300, 0x0301})            # combining grave / acute
+_ARABIC_MARKS = frozenset(range(0x064B, 0x0660)) | frozenset({0x0670})  # tashkeel + dagger alef
+_HEBREW_MARKS = (frozenset(range(0x05B0, 0x05BE))
+                 | frozenset({0x05BF, 0x05C1, 0x05C2, 0x05C4, 0x05C5, 0x05C7}))  # niqqud
+
+
+def _overlay_marks(lang: str) -> "frozenset | None":
+    """The overlay codepoints for *lang*'s diacritization backend, or None.
+
+    Uses exact primary-subtag matching (never ``startswith``) so Aragonese
+    (``arg``), Herero (``her``), Mapudungun (``arn``) etc. are NOT misread as
+    Arabic/Hebrew and stripped.
+    """
+    p = _primary_subtag(lang)
+    if p in STRESS_LANGS:
+        return _STRESS_MARKS
+    if p == "ar":
+        return _ARABIC_MARKS
+    if p == "he":
+        return _HEBREW_MARKS
+    return None
+
+
+def _supports_strip(lang: str) -> bool:
+    """True for languages whose diacritics are removable overlays (stress marks,
+    Arabic tashkeel, Hebrew niqqud) rather than native orthography."""
+    return _overlay_marks(lang) is not None
+
+
+def strip_diacritics(text: str, lang: str = "und", **_) -> str:
+    """Remove the overlay diacritics *lang*'s backend adds, recovering the bare
+    text WITHOUT touching native letters.
+
+    Removes only the specific overlay codepoints (combining acute/grave for
+    stress, tashkeel for Arabic, niqqud for Hebrew) — precomposed native
+    letters (Cyrillic й/ё, Latvian ī, Azerbaijani ç, Arabic hamza carriers) are
+    left intact. Raises :class:`ValueError` for languages whose diacritics are
+    part of the native orthography (e.g. European Portuguese), where any removal
+    would corrupt the spelling.
+    """
+    marks = _overlay_marks(lang)
+    if marks is None:
+        raise ValueError(
+            f"cannot strip diacritics for lang={lang!r}: its diacritics are "
+            "part of the native orthography, so removal would corrupt the "
+            "spelling. strip is defined only for overlay diacritics "
+            "(Arabic/Hebrew vocalization, East-Slavic/Turkic stress).")
+    return "".join(c for c in text if ord(c) not in marks)
+
+
+def register(graph) -> None:
+    """Opt-in graph integration: add the diacritize/strip edge pair.
+
+    ``text -> text-diacritized`` is model-based (``lossless=False``), so a
+    direct ``text -> ipa`` phonemization always out-prices the detour —
+    enabling this extension is safe and non-invasive.  Pair with
+    :func:`scriptconv.phonemizers.register` to make
+    ``"text-diacritized" -> "ipa"`` reachable.
+
+    ``text-diacritized -> text`` (:func:`strip_diacritics`) is lossless and
+    cheap, but gated: it raises for languages whose diacritics are native
+    orthography rather than removable overlays.
+    """
+    graph.register(
+        Edge("text", DIACRITIZED,
+             lambda text, lang="und", model=None, **c: diacritize(text, lang, model, **c),
+             lossless=False))
+    graph.register(
+        Edge(DIACRITIZED, "text",
+             lambda text, lang="und", **_: strip_diacritics(text, lang),
+             lossless=True))

{scriptconv-0.0.4a3 → scriptconv-0.0.4a4}/scriptconv/phonemizers/base.py

@@ -199,9 +199,9 @@ class BasePhonemizer(metaclass=abc.ABCMeta):
         ``ImportError`` naming its extra when the optional dependency is
         missing — scriptconv never installs anything on the caller's behalf.
         """
-        if lang.startswith("he"):
+        if _primary_subtag(lang) == "he":
             return self.phonikud.add_diacritics(text)
-        elif lang.startswith("ar"):
+        elif _primary_subtag(lang) == "ar":
             return self.tashkeel(model).diacritize(text)
         elif _primary_subtag(lang) in STRESS_LANGS:
             return self._stress(text, lang, model)

{scriptconv-0.0.4a3 → scriptconv-0.0.4a4}/scriptconv/phonemizers/registry.py

@@ -219,7 +219,9 @@ def register(graph) -> None:
     routing context, and present only in graphs that opted in (the
     :data:`scriptconv.graph.DEFAULT_GRAPH` stays orthography-only by
     design) — and one dispatching ``text -> ipa`` edge that resolves the
-    per-language default (honouring an ``override=`` context key).
+    per-language default (honouring an ``override=`` context key). It also
+    accepts already-diacritized input via a ``"text-diacritized" -> "ipa"``
+    edge (the same phonemization; pair with :func:`scriptconv.diacritics.register`).
 
     Usage::
 
@@ -236,3 +238,4 @@ def register(graph) -> None:
                                    override).phonemize_string(text, lang)
 
     graph.register(Edge("text", "ipa", _text_to_ipa, lossless=False))
+    graph.register(Edge("text-diacritized", "ipa", _text_to_ipa, lossless=False))

{scriptconv-0.0.4a3 → scriptconv-0.0.4a4}/scriptconv/version.py

@@ -2,7 +2,7 @@
 VERSION_MAJOR = 0
 VERSION_MINOR = 0
 VERSION_BUILD = 4
-VERSION_ALPHA = 3
+VERSION_ALPHA = 4
 # END_VERSION_BLOCK
 
 VERSION_STR = f"{VERSION_MAJOR}.{VERSION_MINOR}.{VERSION_BUILD}"

{scriptconv-0.0.4a3 → scriptconv-0.0.4a4/scriptconv.egg-info}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: scriptconv
-Version: 0.0.4a3
+Version: 0.0.4a4
 Summary: Zero-dependency script & phoneme-notation core — ISO-15924 detection & metadata, IPA↔ARPABET/X-SAMPA/Lexique/Kirshenbaum/Cotovía/RFE, Buckwalter↔Arabic, Hangul→jamo, kana
 License: Apache-2.0
 Project-URL: Homepage, https://github.com/TigreGotico/scriptconv
@@ -391,6 +391,17 @@ p = GraphemePhonemizer()
 p.add_diacritics("Tenho muita sede hoje.", "pt")   # 'Tenho muita sêde hoje.' (thirst)
 ```
 
+Diacritization also joins the graph, like phonemization, via
+`scriptconv.diacritics.register` — a `text -> text-diacritized` edge, opt-in,
+with `text -> ipa` routing unchanged:
+
+```python
+from scriptconv import diacritics
+g = DEFAULT_GRAPH.extend(diacritics.register).extend(phonemizers.register)
+g.convert("Tenho muita sede hoje.", "text", "text-diacritized", lang="pt")
+# 'Tenho muita sêde hoje.'
+```
+
 Stress is unwritten or under-marked in all 26 covered languages; East Slavic
 is the clearest case, where unstressed vowels also reduce (e.g. Russian
 о→[ɐ]/[ə]), so a missing mark corrupts more than prosody there. stressonnx is

{scriptconv-0.0.4a3 → scriptconv-0.0.4a4}/scriptconv.egg-info/SOURCES.txt

@@ -6,6 +6,7 @@ scriptconv/__init__.py
 scriptconv/__main__.py
 scriptconv/cangjie.py
 scriptconv/conventions.py
+scriptconv/diacritics.py
 scriptconv/graph.py
 scriptconv/notation.py
 scriptconv/py.typed
@@ -77,6 +78,7 @@ tests/test_arpa_stress.py
 tests/test_cangjie.py
 tests/test_cli.py
 tests/test_conventions.py
+tests/test_diacritics_graph.py
 tests/test_errors_policy.py
 tests/test_examples.py
 tests/test_graph.py

scriptconv-0.0.4a4/tests/test_diacritics_graph.py

@@ -0,0 +1,104 @@
+import sys
+import types
+import unittest
+from unittest import mock
+
+from scriptconv.graph import DEFAULT_GRAPH
+from scriptconv import diacritics, phonemizers
+
+
+class TestDiacriticsGraphExtension(unittest.TestCase):
+    def setUp(self):
+        self.graph = DEFAULT_GRAPH.extend(diacritics.register).extend(phonemizers.register)
+
+    def test_default_graph_has_no_diacritized_node(self):
+        self.assertFalse(DEFAULT_GRAPH.can_convert("text", "text-diacritized"))
+
+    def test_extend_adds_diacritized_node(self):
+        self.assertTrue(self.graph.can_convert("text", "text-diacritized"))
+        self.assertTrue(self.graph.can_convert("text-diacritized", "ipa"))
+
+    def test_text_to_ipa_prefers_direct_route(self):
+        route = self.graph.route("text", "ipa")
+        self.assertEqual(len(route), 1)
+        edge = route[0]
+        self.assertEqual(edge.src, "text")
+        self.assertEqual(edge.dst, "ipa")
+
+    def test_portuguese_diacritization_via_graph(self):
+        out = self.graph.convert("Tenho muita sede hoje.", "text",
+                                  "text-diacritized", lang="pt")
+        self.assertEqual(out, "Tenho muita sêde hoje.")
+        out2 = self.graph.convert("A sede da empresa fica em Lisboa.", "text",
+                                   "text-diacritized", lang="pt")
+        self.assertEqual(out2, "A séde da empresa fica em Lisboa.")
+
+    def test_slavic_diacritization_routes_to_stressonnx_stub(self):
+        calls = []
+        mod = types.ModuleType("stressonnx")
+
+        def stress(text, lang, model=None):
+            calls.append((text, lang, model))
+            return "STRESSED"
+
+        mod.stress = stress
+        with mock.patch.dict(sys.modules, {"stressonnx": mod}):
+            out = self.graph.convert("замок стоит", "text",
+                                      "text-diacritized", lang="ru")
+        self.assertEqual(out, "STRESSED")
+        self.assertEqual(calls, [("замок стоит", "ru", None)])
+
+    def test_strip_recovers_bare_russian(self):
+        out = self.graph.convert("за́мок", "text-diacritized", "text", lang="ru")
+        self.assertEqual(out, "замок")
+
+    def test_strip_recovers_bare_arabic(self):
+        vocalized = "مُحَمَّد"
+        bare = "".join(c for c in vocalized if not (0x064B <= ord(c) <= 0x065F
+                                                     or ord(c) == 0x0670))
+        out = self.graph.convert(vocalized, "text-diacritized", "text", lang="ar")
+        self.assertEqual(out, bare)
+        self.assertEqual(out, "محمد")
+
+    def test_strip_refused_for_portuguese(self):
+        with self.assertRaises(ValueError) as ctx:
+            self.graph.convert("sêde", "text-diacritized", "text", lang="pt")
+        self.assertIn("native orthography", str(ctx.exception))
+
+    def test_strip_edge_does_not_change_text_to_ipa_route(self):
+        route = self.graph.route("text", "ipa")
+        self.assertEqual(len(route), 1)
+        edge = route[0]
+        self.assertEqual(edge.src, "text")
+        self.assertEqual(edge.dst, "ipa")
+
+    def test_strip_preserves_cyrillic_native_letters(self):
+        out = self.graph.convert("мой родно́й край", "text-diacritized",
+                                  "text", lang="ru")
+        self.assertEqual(out, "мой родной край")
+        out2 = self.graph.convert("ёлка", "text-diacritized", "text", lang="ru")
+        self.assertEqual(out2, "ёлка")
+
+    def test_strip_preserves_latin_stress_lang_diacritics(self):
+        # native macron (ī) plus an added combining acute overlay
+        out = self.graph.convert("Rī́ga", "text-diacritized", "text",
+                                  lang="lv")
+        self.assertEqual(out, "Rīga")
+        out2 = self.graph.convert("Rīga", "text-diacritized", "text", lang="lv")
+        self.assertEqual(out2, "Rīga")
+
+    def test_strip_preserves_arabic_hamza(self):
+        out = self.graph.convert("أَحْمَد", "text-diacritized", "text", lang="ar")
+        self.assertEqual(out, "أحمد")
+
+    def test_strip_refuses_aragonese(self):
+        with self.assertRaises(ValueError):
+            self.graph.convert("Cristián", "text-diacritized", "text", lang="arg")
+
+    def test_add_diacritics_does_not_misroute_herero(self):
+        from scriptconv.phonemizers.base import GraphemePhonemizer
+        self.assertEqual(GraphemePhonemizer().add_diacritics("teste", "her"), "teste")
+
+
+if __name__ == "__main__":
+    unittest.main()