scriptconv 0.0.4a6__tar.gz → 0.0.4a8__tar.gz

{scriptconv-0.0.4a6/scriptconv.egg-info → scriptconv-0.0.4a8}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: scriptconv
-Version: 0.0.4a6
+Version: 0.0.4a8
 Summary: Zero-dependency script & phoneme-notation core — ISO-15924 detection & metadata, IPA↔ARPABET/X-SAMPA/Lexique/Kirshenbaum/Cotovía/RFE, Buckwalter↔Arabic, Hangul→jamo, kana
 License: Apache-2.0
 Project-URL: Homepage, https://github.com/TigreGotico/scriptconv
@@ -373,9 +373,13 @@ Two design points worth knowing:
   phonemizing; that needs language resources scriptconv doesn't ship. Pass
   `normalizer=` (a `(text, lang) -> str` callable) to run yours inside the
   pipeline; without it, text is phonemized as-is.
-- **Model-backed engines never download.** ByT5/Charsiu require an explicit
-  local `model=` path; resolving and caching model files is the caller's
-  concern.
+- **Large/licensed model-backed engines never download.** ByT5/Charsiu
+  require an explicit local `model=` path; resolving and caching those model
+  files is the caller's concern. Small, known-good, unencumbered models are
+  the exception: the Hebrew phonikud diacritizer auto-provisions its ONNX
+  model to a cache dir on first use (`phonikud_model=` still overrides it
+  with a path or callable; cache location via `SCRIPTCONV_CACHE`/
+  `XDG_CACHE_HOME`).
 
 **Pre-G2P disambiguation.** `add_diacritics(text, lang, model=None)` restores
 information ordinary orthography omits but a G2P needs, before phonemization:

{scriptconv-0.0.4a6 → scriptconv-0.0.4a8}/README.md

@@ -246,9 +246,13 @@ Two design points worth knowing:
   phonemizing; that needs language resources scriptconv doesn't ship. Pass
   `normalizer=` (a `(text, lang) -> str` callable) to run yours inside the
   pipeline; without it, text is phonemized as-is.
-- **Model-backed engines never download.** ByT5/Charsiu require an explicit
-  local `model=` path; resolving and caching model files is the caller's
-  concern.
+- **Large/licensed model-backed engines never download.** ByT5/Charsiu
+  require an explicit local `model=` path; resolving and caching those model
+  files is the caller's concern. Small, known-good, unencumbered models are
+  the exception: the Hebrew phonikud diacritizer auto-provisions its ONNX
+  model to a cache dir on first use (`phonikud_model=` still overrides it
+  with a path or callable; cache location via `SCRIPTCONV_CACHE`/
+  `XDG_CACHE_HOME`).
 
 **Pre-G2P disambiguation.** `add_diacritics(text, lang, model=None)` restores
 information ordinary orthography omits but a G2P needs, before phonemization:

{scriptconv-0.0.4a6 → scriptconv-0.0.4a8}/scriptconv/diacritics.py

@@ -51,6 +51,10 @@ tracks which backend/model produced the marks: overlay backends (phonikud,
 tashkeel, stressonnx) are strippable; spelling-integral backends (bifonia) are
 not.
 """
+import os
+import tempfile
+import urllib.request
+from pathlib import Path
 from typing import Optional
 
 from scriptconv.graph import Edge
@@ -119,20 +123,61 @@ _DEFAULT_DIACRITIZER_MODEL = "rawi-ensemble"
 _PHONIKUD_CACHE: dict = {}   # resolved model path/string -> Phonikud instance
 _TASHKEEL_CACHE: dict = {}  # model name -> text2tashkeel Diacritizer
 
+# phonikud is a small (~100MB), public, unlicensed-restriction ONNX model, so
+# — unlike the large/licensed models behind mul.py's ByT5/Charsiu backends —
+# scriptconv auto-provisions it: no local path is required unless the caller
+# wants to override the cache (e.g. an air-gapped host).
+_PHONIKUD_URL = "https://huggingface.co/thewh1teagle/phonikud-onnx/resolve/main/phonikud-1.0.int8.onnx"
 
-def _phonikud(phonikud_model):
+
+def _default_phonikud_model() -> str:
+    """Resolve (downloading and caching on first use if needed) the path to
+    the default phonikud ONNX model.
+
+    Cache directory: ``<base>/scriptconv/phonikud`` where ``<base>`` is
+    ``$SCRIPTCONV_CACHE`` if set, else ``$XDG_CACHE_HOME`` (default
+    ``~/.cache``) — i.e. ``~/.cache/scriptconv/phonikud`` by default. The
+    download is written to a temp file
+    in the same directory and atomically moved into place via
+    :func:`os.replace`, so a failed or interrupted download never leaves a
+    partial file at the destination path.
+    """
+    base = os.environ.get("SCRIPTCONV_CACHE") or os.environ.get(
+        "XDG_CACHE_HOME", os.path.expanduser("~/.cache"))
+    cache_dir = Path(base) / "scriptconv" / "phonikud"
+    dest = cache_dir / "phonikud-1.0.int8.onnx"
+    if not dest.is_file():
+        cache_dir.mkdir(parents=True, exist_ok=True)
+        fd, tmp_path = tempfile.mkstemp(dir=str(cache_dir), prefix=".phonikud-", suffix=".tmp")
+        try:
+            with os.fdopen(fd, "wb") as tmp_f, urllib.request.urlopen(_PHONIKUD_URL) as resp:
+                while True:
+                    chunk = resp.read(1024 * 1024)
+                    if not chunk:
+                        break
+                    tmp_f.write(chunk)
+            os.replace(tmp_path, dest)
+        except BaseException:
+            if os.path.exists(tmp_path):
+                os.remove(tmp_path)
+            raise
+    return str(dest)
+
+
+def _phonikud(phonikud_model=None):
     """Lazily build (and cache) the phonikud Phonikud instance used for Hebrew.
 
-    ``phonikud_model`` is a local path to a phonikud ONNX model, or a
-    zero-arg callable resolving one lazily; scriptconv never downloads —
-    the consumer resolves the file. Install with ``pip install
-    scriptconv[he]`` (or ``pip install phonikud-onnx``)."""
+    ``phonikud_model`` is optional: a local path to a phonikud ONNX model, or
+    a zero-arg callable resolving one lazily. When omitted, scriptconv
+    auto-provisions the small public phonikud model, downloading it once into
+    a cache dir (see :func:`_default_phonikud_model`) and reusing it on
+    subsequent calls. Pass an explicit path/callable to override — e.g. to
+    point at a model already on disk, or on an air-gapped host. Install
+    phonikud-onnx with ``pip install scriptconv[he]`` (or ``pip install
+    phonikud-onnx``)."""
     model = phonikud_model() if callable(phonikud_model) else phonikud_model
     if not model:
-        raise ValueError(
-            "Hebrew diacritization needs a local phonikud ONNX model: "
-            "pass phonikud_model=<path> (scriptconv never downloads "
-            "models; obtain one from the phonikud-onnx release)")
+        model = _default_phonikud_model()
     if model not in _PHONIKUD_CACHE:
         try:
             from phonikud_onnx import Phonikud
@@ -200,21 +245,27 @@ def _sense_diacritics_pt(text: str) -> str:
     return add_extra_diacritics(text)
 
 
-def diacritize(text: str, lang: str = "und", model=None,
-               phonikud_model=None, diacritizer_model=None, **_) -> str:
+def diacritize(text: str, lang: str = "und", diacritizer_model=None,
+               **kwargs) -> str:
     """Add pronunciation-disambiguating diacritics to *text* for *lang*.
 
-    Four backends, each restoring information ordinary orthography omits but
-    downstream G2P needs:
-
-    - Hebrew (``he``) — niqqud via phonikud (``phonikud_model=``).
-    - Arabic (``ar``) — tashkeel via text2tashkeel (``[tashkeel]``,
-      ``model=``/``diacritizer_model=``).
+    ``diacritizer_model=`` is the one model knob for the diacritizer edge — the
+    model for whichever backend the language routes to (there is one such knob,
+    parallel to ``phonemizer_model=`` on the ``text -> ipa`` phonemizer edge):
+
+    - Hebrew (``he``) — niqqud via phonikud; ``diacritizer_model`` is the
+      phonikud ONNX path (or a zero-arg callable). Optional: when omitted the
+      small public phonikud model is auto-downloaded and cached
+      (``$SCRIPTCONV_CACHE``/``$XDG_CACHE_HOME``).
+    - Arabic (``ar``) — tashkeel via text2tashkeel (``[tashkeel]``);
+      ``diacritizer_model`` is the text2tashkeel model name (defaults to
+      ``rawi-ensemble``).
     - East Slavic, Bulgarian/Macedonian/Slovene, Latvian, Armenian, Georgian,
       and Turkic/Caucasian languages (``STRESS_LANGS``, 26 stressonnx tags) —
-      word stress via stressonnx (``[stress]``); stress is unwritten or
-      under-marked in these languages, and in East Slavic unstressed vowels
-      also reduce, so a missing mark can corrupt more than prosody.
+      word stress via stressonnx (``[stress]``); ``diacritizer_model`` is the
+      stressonnx model. Stress is unwritten or under-marked in these languages,
+      and in East Slavic unstressed vowels also reduce, so a missing mark can
+      corrupt more than prosody.
     - European Portuguese (``pt``/``pt-PT``, never ``pt-BR``) —
       heterophonic-homograph sense diacritics via bifonia (``[pt]``);
       ordinary Portuguese orthographic marks that any downstream G2P reads
@@ -224,13 +275,17 @@ def diacritize(text: str, lang: str = "und", model=None,
     ``ImportError`` naming its extra when the optional dependency is missing
     — scriptconv never installs anything on the caller's behalf.
     """
+    # one knob for the diacritizer edge; fold the pre-unification kwargs so
+    # older callers keep working (phonikud_model was the Hebrew-only name).
+    diacritizer_model = (diacritizer_model or kwargs.get("phonikud_model")
+                         or kwargs.get("model"))
     family = _diacritizer_family(lang)
     if family == "he":
-        return _phonikud(phonikud_model).add_diacritics(text)
+        return _phonikud(diacritizer_model).add_diacritics(text)
     if family == "ar":
-        return _tashkeel(model or diacritizer_model).diacritize(text)
+        return _tashkeel(diacritizer_model).diacritize(text)
     if family == "stress":
-        return _stress(text, lang, model)
+        return _stress(text, lang, diacritizer_model)
     if family == "pt":
         return _sense_diacritics_pt(text)
     return text
@@ -311,7 +366,8 @@ def register(graph) -> None:
     """
     graph.register(
         Edge("text", DIACRITIZED,
-             lambda text, lang="und", model=None, **c: diacritize(text, lang, model, **c),
+             lambda text, lang="und", diacritizer_model=None, **c:
+                 diacritize(text, lang, diacritizer_model=diacritizer_model, **c),
              lossless=False))
     graph.register(
         Edge(DIACRITIZED, "text",

{scriptconv-0.0.4a6 → scriptconv-0.0.4a8}/scriptconv/phonemizers/registry.py

@@ -231,9 +231,13 @@ def register(graph) -> None:
     from scriptconv.graph import Edge
 
     def _text_to_ipa(text: str, lang: str = "und",
-                     override: Optional[Phonemizer] = None, **_):
-        return phonemizer_for_lang(lang, Alphabet.IPA,
-                                   override).phonemize_string(text, lang)
+                     override: Optional[Phonemizer] = None,
+                     phonemizer_model: Optional[str] = None, **_):
+        # phonemizer_model is the model knob for this (phonemizer) edge —
+        # forwarded to model-backed phonemizers (ByT5/Charsiu/…); parallel to
+        # diacritizer_model on the text -> text-diacritized edge.
+        return phonemizer_for_lang(lang, Alphabet.IPA, override,
+                                   phonemizer_model).phonemize_string(text, lang)
 
     graph.register(Edge("text", "ipa", _text_to_ipa, lossless=False))
     graph.register(Edge("text-diacritized", "ipa", _text_to_ipa, lossless=False))

{scriptconv-0.0.4a6 → scriptconv-0.0.4a8}/scriptconv/version.py

@@ -2,7 +2,7 @@
 VERSION_MAJOR = 0
 VERSION_MINOR = 0
 VERSION_BUILD = 4
-VERSION_ALPHA = 6
+VERSION_ALPHA = 8
 # END_VERSION_BLOCK
 
 VERSION_STR = f"{VERSION_MAJOR}.{VERSION_MINOR}.{VERSION_BUILD}"

{scriptconv-0.0.4a6 → scriptconv-0.0.4a8/scriptconv.egg-info}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: scriptconv
-Version: 0.0.4a6
+Version: 0.0.4a8
 Summary: Zero-dependency script & phoneme-notation core — ISO-15924 detection & metadata, IPA↔ARPABET/X-SAMPA/Lexique/Kirshenbaum/Cotovía/RFE, Buckwalter↔Arabic, Hangul→jamo, kana
 License: Apache-2.0
 Project-URL: Homepage, https://github.com/TigreGotico/scriptconv
@@ -373,9 +373,13 @@ Two design points worth knowing:
   phonemizing; that needs language resources scriptconv doesn't ship. Pass
   `normalizer=` (a `(text, lang) -> str` callable) to run yours inside the
   pipeline; without it, text is phonemized as-is.
-- **Model-backed engines never download.** ByT5/Charsiu require an explicit
-  local `model=` path; resolving and caching model files is the caller's
-  concern.
+- **Large/licensed model-backed engines never download.** ByT5/Charsiu
+  require an explicit local `model=` path; resolving and caching those model
+  files is the caller's concern. Small, known-good, unencumbered models are
+  the exception: the Hebrew phonikud diacritizer auto-provisions its ONNX
+  model to a cache dir on first use (`phonikud_model=` still overrides it
+  with a path or callable; cache location via `SCRIPTCONV_CACHE`/
+  `XDG_CACHE_HOME`).
 
 **Pre-G2P disambiguation.** `add_diacritics(text, lang, model=None)` restores
 information ordinary orthography omits but a G2P needs, before phonemization:

{scriptconv-0.0.4a6 → scriptconv-0.0.4a8}/tests/test_diacritics.py

@@ -3,6 +3,8 @@
 No optional backend is required — strip_diacritics/_overlay_marks/_supports_strip
 never load a phonemizer model.
 """
+import os
+
 import pytest
 
 from scriptconv.diacritics import (
@@ -90,27 +92,120 @@ def test_overlay_marks_none_for_native_orthography():
     assert _overlay_marks("en") is None
 
 
-class TestHebrewDiacritizerRequiresLocalModel:
-    def test_hebrew_diacritizer_requires_local_model(self):
-        from scriptconv.diacritics import diacritize
-        with pytest.raises(ValueError) as ctx:
-            diacritize("שלום", "he")
-        assert "phonikud_model" in str(ctx.value)
+class TestHebrewDiacritizerAutoProvisions:
+    def test_no_model_given_auto_provisions(self, monkeypatch):
+        from scriptconv import diacritics
+
+        monkeypatch.setitem(diacritics._PHONIKUD_CACHE, "/sentinel/path.onnx", object())
+        monkeypatch.setattr(diacritics, "_default_phonikud_model",
+                             lambda: "/sentinel/path.onnx")
+        result = diacritics._phonikud(None)
+        assert result is diacritics._PHONIKUD_CACHE["/sentinel/path.onnx"]
+
+    def test_explicit_path_bypasses_provisioner(self, monkeypatch):
+        from scriptconv import diacritics
+
+        calls = []
+        monkeypatch.setattr(diacritics, "_default_phonikud_model",
+                             lambda: calls.append(1) or "/should/not/be/used.onnx")
+
+        class FakePhonikud:
+            def __init__(self, model):
+                self.model = model
+
+        monkeypatch.setattr(diacritics, "_PHONIKUD_CACHE", {})
+        import sys
+        import types
+        fake_mod = types.ModuleType("phonikud_onnx")
+        fake_mod.Phonikud = FakePhonikud
+        monkeypatch.setitem(sys.modules, "phonikud_onnx", fake_mod)
+
+        result = diacritics._phonikud("/explicit/path.onnx")
+        assert result.model == "/explicit/path.onnx"
+        assert calls == []  # provisioner never invoked
 
 
 class TestPhonikudModelResolver:
-    def test_callable_resolver_invoked_lazily(self):
-        from scriptconv.diacritics import diacritize
+    def test_callable_resolver_invoked_lazily(self, monkeypatch):
+        from scriptconv import diacritics
+
         calls = []
 
         def resolver():
             calls.append(1)
-            return ""  # resolves to nothing -> still the explicit ValueError
+            return "/from/callable.onnx"
+
+        class FakePhonikud:
+            def __init__(self, model):
+                self.model = model
+
+        monkeypatch.setattr(diacritics, "_PHONIKUD_CACHE", {})
+        import sys
+        import types
+        fake_mod = types.ModuleType("phonikud_onnx")
+        fake_mod.Phonikud = FakePhonikud
+        monkeypatch.setitem(sys.modules, "phonikud_onnx", fake_mod)
 
         assert calls == []  # not resolved at construction
-        with pytest.raises(ValueError):
-            diacritize("שלום", "he", phonikud_model=resolver)
+        result = diacritics._phonikud(resolver)
         assert calls == [1]
+        assert result.model == "/from/callable.onnx"
+
+
+class TestDefaultPhonikudModelProvisioning:
+    def test_downloads_to_cache_dir_and_returns_path(self, monkeypatch, tmp_path):
+        from scriptconv import diacritics
+
+        monkeypatch.setenv("SCRIPTCONV_CACHE", str(tmp_path))
+        calls = []
+
+        class FakeResponse:
+            def __enter__(self):
+                return self
+
+            def __exit__(self, *a):
+                return False
+
+            def read(self, n):
+                if calls and calls[-1] == "read-done":
+                    return b""
+                calls.append("read-done")
+                return b"dummy-onnx-bytes"
+
+        def fake_urlopen(url):
+            calls.append(("urlopen", url))
+            return FakeResponse()
+
+        monkeypatch.setattr(diacritics.urllib.request, "urlopen", fake_urlopen)
+
+        path = diacritics._default_phonikud_model()
+        assert path == str(tmp_path / "scriptconv" / "phonikud" / "phonikud-1.0.int8.onnx")
+        assert os.path.isfile(path)
+
+        # idempotent: second call must not re-download
+        urlopen_calls_before = sum(1 for c in calls if isinstance(c, tuple))
+        diacritics._default_phonikud_model()
+        urlopen_calls_after = sum(1 for c in calls if isinstance(c, tuple))
+        assert urlopen_calls_before == urlopen_calls_after == 1
+
+    def test_failed_download_leaves_no_partial_file(self, monkeypatch, tmp_path):
+        from scriptconv import diacritics
+
+        monkeypatch.setenv("SCRIPTCONV_CACHE", str(tmp_path))
+
+        def failing_urlopen(url):
+            raise OSError("network unavailable")
+
+        monkeypatch.setattr(diacritics.urllib.request, "urlopen", failing_urlopen)
+
+        with pytest.raises(OSError):
+            diacritics._default_phonikud_model()
+
+        dest = tmp_path / "scriptconv" / "phonikud" / "phonikud-1.0.int8.onnx"
+        assert not dest.exists()
+        cache_dir = tmp_path / "scriptconv" / "phonikud"
+        if cache_dir.exists():
+            assert list(cache_dir.iterdir()) == []
 
 
 class TestEuropeanPortugueseSenseDiacritics:

{scriptconv-0.0.4a6 → scriptconv-0.0.4a8}/tests/test_diacritics_graph.py

@@ -101,3 +101,49 @@ class TestDiacriticsGraphExtension(unittest.TestCase):
 
 if __name__ == "__main__":
     unittest.main()
+
+
+class TestEdgeModelKeys(unittest.TestCase):
+    """The two model-bearing edges each read their own context key: the
+    diacritizer edge (text -> text-diacritized) reads ``diacritizer_model``,
+    the phonemizer edge (text -> ipa) reads ``phonemizer_model``. Neither picks
+    up the other's key."""
+
+    def setUp(self):
+        self.graph = DEFAULT_GRAPH.extend(diacritics.register).extend(phonemizers.register)
+
+    def test_diacritizer_edge_reads_diacritizer_model(self):
+        seen = {}
+        backend = types.SimpleNamespace(diacritize=lambda t: t)
+
+        def _spy(m=None):
+            seen["m"] = m
+            return backend
+
+        with mock.patch.object(diacritics, "_tashkeel", _spy):
+            self.graph.convert("نص", "text", "text-diacritized", lang="ar",
+                               diacritizer_model="RAWI-X",
+                               phonemizer_model="IGNORED")
+        self.assertEqual(seen["m"], "RAWI-X")
+
+    def test_phonemizer_edge_reads_phonemizer_model(self):
+        from scriptconv.phonemizers import registry
+        cap = {}
+
+        class _P:
+            def phonemize_string(self, t, l):
+                return "IPA"
+
+        def _spy(lang, alphabet, override=None, model=None, **k):
+            cap["model"] = model
+            return _P()
+
+        with mock.patch.object(registry, "phonemizer_for_lang", _spy):
+            self.graph.convert("hello", "text", "ipa", lang="en",
+                               phonemizer_model="BYT5-CKPT",
+                               diacritizer_model="IGNORED")
+        self.assertEqual(cap["model"], "BYT5-CKPT")
+
+
+if __name__ == "__main__":
+    unittest.main()