scriptconv 0.0.4a6__tar.gz → 0.0.4a7__tar.gz

{scriptconv-0.0.4a6/scriptconv.egg-info → scriptconv-0.0.4a7}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: scriptconv
-Version: 0.0.4a6
+Version: 0.0.4a7
 Summary: Zero-dependency script & phoneme-notation core — ISO-15924 detection & metadata, IPA↔ARPABET/X-SAMPA/Lexique/Kirshenbaum/Cotovía/RFE, Buckwalter↔Arabic, Hangul→jamo, kana
 License: Apache-2.0
 Project-URL: Homepage, https://github.com/TigreGotico/scriptconv
@@ -373,9 +373,13 @@ Two design points worth knowing:
   phonemizing; that needs language resources scriptconv doesn't ship. Pass
   `normalizer=` (a `(text, lang) -> str` callable) to run yours inside the
   pipeline; without it, text is phonemized as-is.
-- **Model-backed engines never download.** ByT5/Charsiu require an explicit
-  local `model=` path; resolving and caching model files is the caller's
-  concern.
+- **Large/licensed model-backed engines never download.** ByT5/Charsiu
+  require an explicit local `model=` path; resolving and caching those model
+  files is the caller's concern. Small, known-good, unencumbered models are
+  the exception: the Hebrew phonikud diacritizer auto-provisions its ONNX
+  model to a cache dir on first use (`phonikud_model=` still overrides it
+  with a path or callable; cache location via `SCRIPTCONV_CACHE`/
+  `XDG_CACHE_HOME`).
 
 **Pre-G2P disambiguation.** `add_diacritics(text, lang, model=None)` restores
 information ordinary orthography omits but a G2P needs, before phonemization:

{scriptconv-0.0.4a6 → scriptconv-0.0.4a7}/README.md

@@ -246,9 +246,13 @@ Two design points worth knowing:
   phonemizing; that needs language resources scriptconv doesn't ship. Pass
   `normalizer=` (a `(text, lang) -> str` callable) to run yours inside the
   pipeline; without it, text is phonemized as-is.
-- **Model-backed engines never download.** ByT5/Charsiu require an explicit
-  local `model=` path; resolving and caching model files is the caller's
-  concern.
+- **Large/licensed model-backed engines never download.** ByT5/Charsiu
+  require an explicit local `model=` path; resolving and caching those model
+  files is the caller's concern. Small, known-good, unencumbered models are
+  the exception: the Hebrew phonikud diacritizer auto-provisions its ONNX
+  model to a cache dir on first use (`phonikud_model=` still overrides it
+  with a path or callable; cache location via `SCRIPTCONV_CACHE`/
+  `XDG_CACHE_HOME`).
 
 **Pre-G2P disambiguation.** `add_diacritics(text, lang, model=None)` restores
 information ordinary orthography omits but a G2P needs, before phonemization:

{scriptconv-0.0.4a6 → scriptconv-0.0.4a7}/scriptconv/diacritics.py

@@ -51,6 +51,10 @@ tracks which backend/model produced the marks: overlay backends (phonikud,
 tashkeel, stressonnx) are strippable; spelling-integral backends (bifonia) are
 not.
 """
+import os
+import tempfile
+import urllib.request
+from pathlib import Path
 from typing import Optional
 
 from scriptconv.graph import Edge
@@ -119,20 +123,61 @@ _DEFAULT_DIACRITIZER_MODEL = "rawi-ensemble"
 _PHONIKUD_CACHE: dict = {}   # resolved model path/string -> Phonikud instance
 _TASHKEEL_CACHE: dict = {}  # model name -> text2tashkeel Diacritizer
 
+# phonikud is a small (~100MB), public, unlicensed-restriction ONNX model, so
+# — unlike the large/licensed models behind mul.py's ByT5/Charsiu backends —
+# scriptconv auto-provisions it: no local path is required unless the caller
+# wants to override the cache (e.g. an air-gapped host).
+_PHONIKUD_URL = "https://huggingface.co/thewh1teagle/phonikud-onnx/resolve/main/phonikud-1.0.int8.onnx"
 
-def _phonikud(phonikud_model):
+
+def _default_phonikud_model() -> str:
+    """Resolve (downloading and caching on first use if needed) the path to
+    the default phonikud ONNX model.
+
+    Cache directory: ``<base>/scriptconv/phonikud`` where ``<base>`` is
+    ``$SCRIPTCONV_CACHE`` if set, else ``$XDG_CACHE_HOME`` (default
+    ``~/.cache``) — i.e. ``~/.cache/scriptconv/phonikud`` by default. The
+    download is written to a temp file
+    in the same directory and atomically moved into place via
+    :func:`os.replace`, so a failed or interrupted download never leaves a
+    partial file at the destination path.
+    """
+    base = os.environ.get("SCRIPTCONV_CACHE") or os.environ.get(
+        "XDG_CACHE_HOME", os.path.expanduser("~/.cache"))
+    cache_dir = Path(base) / "scriptconv" / "phonikud"
+    dest = cache_dir / "phonikud-1.0.int8.onnx"
+    if not dest.is_file():
+        cache_dir.mkdir(parents=True, exist_ok=True)
+        fd, tmp_path = tempfile.mkstemp(dir=str(cache_dir), prefix=".phonikud-", suffix=".tmp")
+        try:
+            with os.fdopen(fd, "wb") as tmp_f, urllib.request.urlopen(_PHONIKUD_URL) as resp:
+                while True:
+                    chunk = resp.read(1024 * 1024)
+                    if not chunk:
+                        break
+                    tmp_f.write(chunk)
+            os.replace(tmp_path, dest)
+        except BaseException:
+            if os.path.exists(tmp_path):
+                os.remove(tmp_path)
+            raise
+    return str(dest)
+
+
+def _phonikud(phonikud_model=None):
     """Lazily build (and cache) the phonikud Phonikud instance used for Hebrew.
 
-    ``phonikud_model`` is a local path to a phonikud ONNX model, or a
-    zero-arg callable resolving one lazily; scriptconv never downloads —
-    the consumer resolves the file. Install with ``pip install
-    scriptconv[he]`` (or ``pip install phonikud-onnx``)."""
+    ``phonikud_model`` is optional: a local path to a phonikud ONNX model, or
+    a zero-arg callable resolving one lazily. When omitted, scriptconv
+    auto-provisions the small public phonikud model, downloading it once into
+    a cache dir (see :func:`_default_phonikud_model`) and reusing it on
+    subsequent calls. Pass an explicit path/callable to override — e.g. to
+    point at a model already on disk, or on an air-gapped host. Install
+    phonikud-onnx with ``pip install scriptconv[he]`` (or ``pip install
+    phonikud-onnx``)."""
     model = phonikud_model() if callable(phonikud_model) else phonikud_model
     if not model:
-        raise ValueError(
-            "Hebrew diacritization needs a local phonikud ONNX model: "
-            "pass phonikud_model=<path> (scriptconv never downloads "
-            "models; obtain one from the phonikud-onnx release)")
+        model = _default_phonikud_model()
     if model not in _PHONIKUD_CACHE:
         try:
             from phonikud_onnx import Phonikud
@@ -207,7 +252,10 @@ def diacritize(text: str, lang: str = "und", model=None,
     Four backends, each restoring information ordinary orthography omits but
     downstream G2P needs:
 
-    - Hebrew (``he``) — niqqud via phonikud (``phonikud_model=``).
+    - Hebrew (``he``) — niqqud via phonikud. ``phonikud_model=`` is optional:
+      omitted, the small public phonikud ONNX model is auto-downloaded and
+      cached (``$SCRIPTCONV_CACHE``/``$XDG_CACHE_HOME``); pass a path or
+      zero-arg callable to override.
     - Arabic (``ar``) — tashkeel via text2tashkeel (``[tashkeel]``,
       ``model=``/``diacritizer_model=``).
     - East Slavic, Bulgarian/Macedonian/Slovene, Latvian, Armenian, Georgian,

{scriptconv-0.0.4a6 → scriptconv-0.0.4a7}/scriptconv/version.py

@@ -2,7 +2,7 @@
 VERSION_MAJOR = 0
 VERSION_MINOR = 0
 VERSION_BUILD = 4
-VERSION_ALPHA = 6
+VERSION_ALPHA = 7
 # END_VERSION_BLOCK
 
 VERSION_STR = f"{VERSION_MAJOR}.{VERSION_MINOR}.{VERSION_BUILD}"

{scriptconv-0.0.4a6 → scriptconv-0.0.4a7/scriptconv.egg-info}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: scriptconv
-Version: 0.0.4a6
+Version: 0.0.4a7
 Summary: Zero-dependency script & phoneme-notation core — ISO-15924 detection & metadata, IPA↔ARPABET/X-SAMPA/Lexique/Kirshenbaum/Cotovía/RFE, Buckwalter↔Arabic, Hangul→jamo, kana
 License: Apache-2.0
 Project-URL: Homepage, https://github.com/TigreGotico/scriptconv
@@ -373,9 +373,13 @@ Two design points worth knowing:
   phonemizing; that needs language resources scriptconv doesn't ship. Pass
   `normalizer=` (a `(text, lang) -> str` callable) to run yours inside the
   pipeline; without it, text is phonemized as-is.
-- **Model-backed engines never download.** ByT5/Charsiu require an explicit
-  local `model=` path; resolving and caching model files is the caller's
-  concern.
+- **Large/licensed model-backed engines never download.** ByT5/Charsiu
+  require an explicit local `model=` path; resolving and caching those model
+  files is the caller's concern. Small, known-good, unencumbered models are
+  the exception: the Hebrew phonikud diacritizer auto-provisions its ONNX
+  model to a cache dir on first use (`phonikud_model=` still overrides it
+  with a path or callable; cache location via `SCRIPTCONV_CACHE`/
+  `XDG_CACHE_HOME`).
 
 **Pre-G2P disambiguation.** `add_diacritics(text, lang, model=None)` restores
 information ordinary orthography omits but a G2P needs, before phonemization:

{scriptconv-0.0.4a6 → scriptconv-0.0.4a7}/tests/test_diacritics.py

@@ -3,6 +3,8 @@
 No optional backend is required — strip_diacritics/_overlay_marks/_supports_strip
 never load a phonemizer model.
 """
+import os
+
 import pytest
 
 from scriptconv.diacritics import (
@@ -90,27 +92,120 @@ def test_overlay_marks_none_for_native_orthography():
     assert _overlay_marks("en") is None
 
 
-class TestHebrewDiacritizerRequiresLocalModel:
-    def test_hebrew_diacritizer_requires_local_model(self):
-        from scriptconv.diacritics import diacritize
-        with pytest.raises(ValueError) as ctx:
-            diacritize("שלום", "he")
-        assert "phonikud_model" in str(ctx.value)
+class TestHebrewDiacritizerAutoProvisions:
+    def test_no_model_given_auto_provisions(self, monkeypatch):
+        from scriptconv import diacritics
+
+        monkeypatch.setitem(diacritics._PHONIKUD_CACHE, "/sentinel/path.onnx", object())
+        monkeypatch.setattr(diacritics, "_default_phonikud_model",
+                             lambda: "/sentinel/path.onnx")
+        result = diacritics._phonikud(None)
+        assert result is diacritics._PHONIKUD_CACHE["/sentinel/path.onnx"]
+
+    def test_explicit_path_bypasses_provisioner(self, monkeypatch):
+        from scriptconv import diacritics
+
+        calls = []
+        monkeypatch.setattr(diacritics, "_default_phonikud_model",
+                             lambda: calls.append(1) or "/should/not/be/used.onnx")
+
+        class FakePhonikud:
+            def __init__(self, model):
+                self.model = model
+
+        monkeypatch.setattr(diacritics, "_PHONIKUD_CACHE", {})
+        import sys
+        import types
+        fake_mod = types.ModuleType("phonikud_onnx")
+        fake_mod.Phonikud = FakePhonikud
+        monkeypatch.setitem(sys.modules, "phonikud_onnx", fake_mod)
+
+        result = diacritics._phonikud("/explicit/path.onnx")
+        assert result.model == "/explicit/path.onnx"
+        assert calls == []  # provisioner never invoked
 
 
 class TestPhonikudModelResolver:
-    def test_callable_resolver_invoked_lazily(self):
-        from scriptconv.diacritics import diacritize
+    def test_callable_resolver_invoked_lazily(self, monkeypatch):
+        from scriptconv import diacritics
+
         calls = []
 
         def resolver():
             calls.append(1)
-            return ""  # resolves to nothing -> still the explicit ValueError
+            return "/from/callable.onnx"
+
+        class FakePhonikud:
+            def __init__(self, model):
+                self.model = model
+
+        monkeypatch.setattr(diacritics, "_PHONIKUD_CACHE", {})
+        import sys
+        import types
+        fake_mod = types.ModuleType("phonikud_onnx")
+        fake_mod.Phonikud = FakePhonikud
+        monkeypatch.setitem(sys.modules, "phonikud_onnx", fake_mod)
 
         assert calls == []  # not resolved at construction
-        with pytest.raises(ValueError):
-            diacritize("שלום", "he", phonikud_model=resolver)
+        result = diacritics._phonikud(resolver)
         assert calls == [1]
+        assert result.model == "/from/callable.onnx"
+
+
+class TestDefaultPhonikudModelProvisioning:
+    def test_downloads_to_cache_dir_and_returns_path(self, monkeypatch, tmp_path):
+        from scriptconv import diacritics
+
+        monkeypatch.setenv("SCRIPTCONV_CACHE", str(tmp_path))
+        calls = []
+
+        class FakeResponse:
+            def __enter__(self):
+                return self
+
+            def __exit__(self, *a):
+                return False
+
+            def read(self, n):
+                if calls and calls[-1] == "read-done":
+                    return b""
+                calls.append("read-done")
+                return b"dummy-onnx-bytes"
+
+        def fake_urlopen(url):
+            calls.append(("urlopen", url))
+            return FakeResponse()
+
+        monkeypatch.setattr(diacritics.urllib.request, "urlopen", fake_urlopen)
+
+        path = diacritics._default_phonikud_model()
+        assert path == str(tmp_path / "scriptconv" / "phonikud" / "phonikud-1.0.int8.onnx")
+        assert os.path.isfile(path)
+
+        # idempotent: second call must not re-download
+        urlopen_calls_before = sum(1 for c in calls if isinstance(c, tuple))
+        diacritics._default_phonikud_model()
+        urlopen_calls_after = sum(1 for c in calls if isinstance(c, tuple))
+        assert urlopen_calls_before == urlopen_calls_after == 1
+
+    def test_failed_download_leaves_no_partial_file(self, monkeypatch, tmp_path):
+        from scriptconv import diacritics
+
+        monkeypatch.setenv("SCRIPTCONV_CACHE", str(tmp_path))
+
+        def failing_urlopen(url):
+            raise OSError("network unavailable")
+
+        monkeypatch.setattr(diacritics.urllib.request, "urlopen", failing_urlopen)
+
+        with pytest.raises(OSError):
+            diacritics._default_phonikud_model()
+
+        dest = tmp_path / "scriptconv" / "phonikud" / "phonikud-1.0.int8.onnx"
+        assert not dest.exists()
+        cache_dir = tmp_path / "scriptconv" / "phonikud"
+        if cache_dir.exists():
+            assert list(cache_dir.iterdir()) == []
 
 
 class TestEuropeanPortugueseSenseDiacritics: