scriptconv 0.0.4a14__tar.gz → 0.0.4a16__tar.gz

{scriptconv-0.0.4a14/scriptconv.egg-info → scriptconv-0.0.4a16}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: scriptconv
-Version: 0.0.4a14
+Version: 0.0.4a16
 Summary: Zero-dependency script & phoneme-notation core — ISO-15924 detection & metadata, IPA↔ARPABET/X-SAMPA/Lexique/Kirshenbaum/Cotovía/RFE, Buckwalter↔Arabic, Hangul→jamo, kana
 License: Apache-2.0
 Project-URL: Homepage, https://github.com/TigreGotico/scriptconv

{scriptconv-0.0.4a14 → scriptconv-0.0.4a16}/scriptconv/notation.py

@@ -154,6 +154,10 @@ for _arpa, _ipa in _ARPA_BASE.items():
         _IPA_TO_ARPA[_ipa] = _arpa
 # Accept ASCII "g" (U+0067) as well as canonical script "ɡ" (U+0261) on input.
 _IPA_TO_ARPA.setdefault("g", "G")
+# Bare/long NURSE vowel (non-rhotic British ɜ, ɜː) has no dedicated ARPA symbol —
+# ARPABET is rhotic-only, so it collapses onto the same "ER" as the r-colored ɜr.
+_IPA_TO_ARPA.setdefault("ɜː", "ER")
+_IPA_TO_ARPA.setdefault("ɜ", "ER")
 
 # Precompiled regex for IPA → ARPA (longest-first)
 _IPA_ARPA_KEYS_SORTED = sorted(_IPA_TO_ARPA.keys(), key=len, reverse=True)
@@ -307,6 +311,7 @@ _XSAMPA_TO_IPA: dict[str, str] = {
     "p\\": "ɸ",
     "B\\": "ʙ",
     "r\\": "ɹ",
+    "r": "r",     # plain r = alveolar TRILL in canonical X-SAMPA (tap is "4")
     "r`": "ɽ",    # retroflex FLAP (not approximant)
     "R\\": "ʀ",
     "l`": "ɭ",

{scriptconv-0.0.4a14 → scriptconv-0.0.4a16}/scriptconv/phonemizers/shami.py

@@ -58,8 +58,21 @@ class ShamiPhonemizer(BasePhonemizer):
 
     def phonemize(self, text: str, lang: str) -> PhonemizedChunks:
         """Return sentence-level phoneme lists."""
-        phonemes, _ = self.phonemize_with_language_ids(text, lang)
-        return phonemes
+        return list(self.phonemize_lazy(text, lang))
+
+    def phonemize_lazy(self, text: str, lang: str) -> Iterator[List[str]]:
+        """Sentence-level lists of ShamiVITS symbols, one sentence at a time.
+
+        The front-end emits the model's own 88-symbol inventory, 24 of which are
+        multi-character (``d\u0361\u0292``, ``t\u0361\u0283``, ``s\u02e4``, the long vowels ``a\u02d0``/``i\u02d0``/``u\u02d0``,
+        the diphthongs ``a\u0361\u026a``/``o\u0361\u028a`` \u2026) plus the ``<bos>``/``<eos>`` sentinels.
+        :meth:`BasePhonemizer.phonemize_lazy` splits the phoneme *string* per
+        character, which would shred ``i\u02d0`` into ``i`` + ``\u02d0`` \u2014 and a bare ``\u02d0`` is
+        not in the inventory, so the tokenizer drops it as OOV and the model is
+        fed a shorter, wrong sequence. Yield the front-end's own tokens instead.
+        """
+        for phonemes, _ in self.phonemize_with_language_ids_lazy(text, lang):
+            yield phonemes
 
     def phonemize_with_language_ids(
         self, text: str, lang: str

{scriptconv-0.0.4a14 → scriptconv-0.0.4a16}/scriptconv/version.py

@@ -2,7 +2,7 @@
 VERSION_MAJOR = 0
 VERSION_MINOR = 0
 VERSION_BUILD = 4
-VERSION_ALPHA = 14
+VERSION_ALPHA = 16
 # END_VERSION_BLOCK
 
 VERSION_STR = f"{VERSION_MAJOR}.{VERSION_MINOR}.{VERSION_BUILD}"

{scriptconv-0.0.4a14 → scriptconv-0.0.4a16/scriptconv.egg-info}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: scriptconv
-Version: 0.0.4a14
+Version: 0.0.4a16
 Summary: Zero-dependency script & phoneme-notation core — ISO-15924 detection & metadata, IPA↔ARPABET/X-SAMPA/Lexique/Kirshenbaum/Cotovía/RFE, Buckwalter↔Arabic, Hangul→jamo, kana
 License: Apache-2.0
 Project-URL: Homepage, https://github.com/TigreGotico/scriptconv

{scriptconv-0.0.4a14 → scriptconv-0.0.4a16}/tests/test_phonemizers_cjk_ar.py

@@ -99,6 +99,33 @@ class TestShamiFrontend(unittest.TestCase):
         lazy = list(p.phonemize_with_language_ids_lazy(text, "ar"))
         self.assertEqual(eager[0], [ph for ph, _ in lazy])
 
+    def test_phonemize_lazy_matches_phonemize(self):
+        """phoonnx (and any streaming caller) consumes phonemize_lazy only."""
+        from scriptconv.phonemizers.shami import ShamiPhonemizer
+        p = ShamiPhonemizer()
+        text = "\u0627\u0644\u062c\u0648 \u062d\u0644\u0648 \u0627\u0644\u064a\u0648\u0645. \u0627\u0644\u0634\u062c\u0631\u0629 \u0643\u0628\u064a\u0631\u0629."
+        self.assertEqual(p.phonemize(text, "ar"),
+                         list(p.phonemize_lazy(text, "ar")))
+
+    def test_phonemize_lazy_keeps_multichar_symbols_whole(self):
+        """Every yielded token is a whole inventory symbol, never a shredded
+        fragment. The exact phoneme sequence depends on which optional Arabic
+        diacritizer backend is installed, so the assertions are invariants over
+        the inventory, not a hardcoded transcription: a bare length mark can
+        only appear if a long-vowel symbol was split, and every token must be
+        a real ShamiVITS symbol."""
+        from scriptconv.phonemizers.shami import ShamiPhonemizer, SYMBOL_TO_ID
+        p = ShamiPhonemizer()
+        tokens = [t for sent in p.phonemize_lazy("\u0627\u0644\u0634\u062c\u0631\u0629 \u0643\u0628\u064a\u0631\u0629.", "ar")
+                  for t in sent]
+        self.assertNotIn("\u02d0", tokens)
+        for t in tokens:
+            self.assertIn(t, SYMBOL_TO_ID, f"{t!r} is not a ShamiVITS symbol")
+        # If any multi-character symbol was produced, it survived whole.
+        multi = [t for t in tokens if len(t) > 1 and not t.startswith("<")]
+        for t in multi:
+            self.assertIn(t, SYMBOL_TO_ID)
+
     def test_frontend_symbols_are_public(self):
         from scriptconv.phonemizers.shami import (
             TextFrontend,