scriptconv 0.0.4a31__tar.gz → 0.0.4a32__tar.gz

{scriptconv-0.0.4a31/scriptconv.egg-info → scriptconv-0.0.4a32}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: scriptconv
-Version: 0.0.4a31
+Version: 0.0.4a32
 Summary: Zero-dependency script & phoneme-notation core — ISO-15924 detection & metadata, IPA↔ARPABET/X-SAMPA/Lexique/Kirshenbaum/Cotovía/RFE, Buckwalter↔Arabic, Hangul→jamo, kana
 License: Apache-2.0
 Project-URL: Homepage, https://github.com/TigreGotico/scriptconv

scriptconv-0.0.4a32/scriptconv/phonemizers/_vendored/matoub_kab/__init__.py

@@ -0,0 +1,143 @@
+"""Vendored Kabyle grapheme-to-phoneme rules from agbalu/Matoub-82M.
+
+Source: https://huggingface.co/agbalu/Matoub-82M, file ``tokenization_matoub.py``
+at revision ``3d00056f3663d4d1d364e9b12c21230685a0ba9a``, sha256
+``c95fdf4bd23efa649287662a0059d9053d17c8cab4fade383784e084cc422416``, the value
+the model's own ``export.stats.json`` records for that file. Licence Apache-2.0,
+the same licence this repository uses. See ``LICENSE.md`` beside this file.
+
+Only the rule functions are kept. The upstream ``MatoubTokenizer`` class is
+dropped: it subclasses ``transformers.PreTrainedTokenizer``, and scriptconv
+emits phoneme strings, not model ids. The ``vocab.json`` constant goes with it.
+The rules themselves are unchanged, because a caller that feeds Matoub-82M needs
+the exact string that model was fitted on.
+
+The upstream file states that these rules are a copy of ``agbalu.tts.g2p`` and
+``agbalu.tts.kokoro``, kept in step by a test in that project. This copy is one
+step further away, so ``tests/test_phonemizers_matoub_kab.py`` holds it to
+measured outputs.
+"""
+
+from __future__ import annotations
+
+import re
+from typing import Final
+
+
+VOWELS: Final = frozenset("aeiou")
+
+BACKING_TRIGGERS: Final = frozenset("ḍṣṭẓṛqɣx")
+
+SPIRANTS: Final[dict[str, tuple[str, str]]] = {
+    "b": ("β", "b"),
+    "d": ("ð", "d"),
+    "g": ("ʝ", "ɡ"),
+    "k": ("ç", "k"),
+    "t": ("θ", "t"),
+    "ḍ": ("ðˤ", "dˤ"),
+}
+
+PLAIN: Final[dict[str, str]] = {
+    "a": "æ",
+    "c": "ʃ",
+    "e": "ə",
+    "f": "f",
+    "h": "h",
+    "i": "i",
+    "j": "ʒ",
+    "l": "l",
+    "m": "m",
+    "n": "n",
+    "o": "o",
+    "p": "p",
+    "q": "q",
+    "r": "r",
+    "s": "s",
+    "u": "u",
+    "v": "v",
+    "w": "w",
+    "x": "χ",
+    "y": "j",
+    "z": "z",
+    "č": "t͡ʃ",
+    "ǧ": "d͡ʒ",
+    "ɛ": "ʕ",
+    "ɣ": "ʁ",
+    "ḥ": "ħ",
+    "ṛ": "rˤ",
+    "ṣ": "sˤ",
+    "ṭ": "tˤ",
+    "ẓ": "zˤ",
+}
+
+NASAL_ASSIMILATION: Final[dict[str, str]] = {"f": "m", "m": "m", "y": "ɲ", "q": "ŋ", "x": "ŋ"}
+
+FOLD: Final[dict[str, str]] = {"t͡ʃ": "ʧ", "d͡ʒ": "ʤ"}
+"""Tie-bar sequences the base model already carries as single symbols."""
+
+LEGACY_TENSE_T: Final = "ţ"
+LENGTH: Final = "ː"
+BOUNDARY: Final = " "
+
+_SPLIT: Final = re.compile(r"[\s\-]+")
+_STRIP: Final = "«»\"'“”‘’.,;:!?()[]{}…"
+
+
+class PhonemeError(ValueError):
+    """A character with no rule."""
+
+
+def _segment(word: str) -> list[tuple[str, bool]]:
+    segments: list[tuple[str, bool]] = []
+    index = 0
+    while index < len(word):
+        char = word[index]
+        paired = index + 1 < len(word) and word[index + 1] == char and char not in VOWELS
+        segments.append((char, paired))
+        index += 2 if paired else 1
+    return segments
+
+
+def _backed(chars: list[str], position: int) -> bool:
+    before = chars[position - 1] if position > 0 else ""
+    after = chars[position + 1] if position + 1 < len(chars) else ""
+    return before in BACKING_TRIGGERS or after in BACKING_TRIGGERS
+
+
+def phonemize_word(word: str) -> str:
+    """One orthographic word to IPA. Raises on any character without a rule."""
+    segments = _segment(word.casefold())
+    chars = [char for char, _ in segments]
+    out: list[str] = []
+    for position, (char, geminate) in enumerate(segments):
+        if char == LEGACY_TENSE_T:
+            out.append(SPIRANTS["t"][1] + LENGTH)
+            continue
+        if char in SPIRANTS:
+            short, stop = SPIRANTS[char]
+            out.append(stop + LENGTH if geminate else short)
+            continue
+        if char not in PLAIN:
+            message = f"no rule for {char!r} (U+{ord(char):04X}) in {word!r}"
+            raise PhonemeError(message)
+        if char == "a" and _backed(chars, position):
+            symbol = "ɑ"
+        elif char == "n" and not geminate and position + 1 < len(chars):
+            symbol = NASAL_ASSIMILATION.get(chars[position + 1], PLAIN["n"])
+        else:
+            symbol = PLAIN[char]
+        out.append(symbol + LENGTH if geminate else symbol)
+    return "".join(out)
+
+
+def fold(ipa: str) -> str:
+    """Rewrite tie-bar affricates onto the base model's own single symbols."""
+    for sequence, symbol in FOLD.items():
+        ipa = ipa.replace(sequence, symbol)
+    return ipa
+
+
+def phonemize(text: str) -> str:
+    """A Kabyle sentence to the phoneme string the model was fitted on."""
+    words = [token.strip(_STRIP) for token in _SPLIT.split(text) if token.strip(_STRIP)]
+    return fold(BOUNDARY.join(phonemize_word(word) for word in words))

scriptconv-0.0.4a32/scriptconv/phonemizers/ber.py

@@ -0,0 +1,62 @@
+"""Berber phonemizers.
+
+Kabyle (``kab``) through the rule set that ``agbalu/Matoub-82M`` was fitted on.
+
+This is deliberately a separate route from :class:`AfricaG2PPhonemizer`, which
+also answers ``kab``. The two are different phonemizers and their output
+differs, so they are not interchangeable:
+
+===============  ===========================  ==============
+text             africa-g2p                   Matoub
+===============  ===========================  ==============
+``taddart``      ``t a ð ð a r t``            ``θædːærθ``
+``ameqqran``     ``a m ə q q r a n``          ``æməqːræn``
+``aqcic``        ``a q ʃ i ʃ``                ``ɑqʃiʃ``
+===============  ===========================  ==============
+
+africa-g2p does not spirantise the initial ``t``, writes gemination as a
+doubled symbol rather than length, and does not back ``/a/`` beside a backing
+consonant. A caller driving Matoub-82M needs the Matoub string: the other one
+carries symbols that model has no embedding row for.
+"""
+from typing import List
+
+from scriptconv.phonemizers.base import BasePhonemizer
+from scriptconv.phonemizers.enums import Alphabet
+
+
+class MatoubKabylePhonemizer(BasePhonemizer):
+    """Kabyle Latin orthography to the IPA string ``agbalu/Matoub-82M`` reads.
+
+    The rules are vendored under ``_vendored/matoub_kab`` (Apache-2.0, the same
+    licence as this repository) and need nothing outside the standard library.
+
+    The rules raise on a character they have no rule for, and that is kept on
+    purpose. The upstream file records that silent deletion already cost three
+    Kabyle consonants once, so an unknown symbol is reported rather than
+    dropped.
+    """
+
+    _LANGS: List[str] = ["kab"]
+
+    def __init__(self):
+        super().__init__(Alphabet.IPA)
+
+    @classmethod
+    def supported_langs(cls) -> List[str]:
+        return list(cls._LANGS)
+
+    @classmethod
+    def get_lang(cls, target_lang: str) -> str:
+        """Validate *target_lang* against the one language these rules cover.
+
+        Raises:
+            ValueError: if the language code is not Kabyle.
+        """
+        return cls.match_lang(target_lang, cls._LANGS)
+
+    def phonemize_string(self, text: str, lang: str = "kab") -> str:
+        self.get_lang(lang)
+        from scriptconv.phonemizers._vendored.matoub_kab import phonemize
+
+        return phonemize(text)

{scriptconv-0.0.4a31 → scriptconv-0.0.4a32}/scriptconv/phonemizers/enums.py

@@ -86,3 +86,4 @@ class Phonemizer(str, Enum):
     ORTHOGRAPHY2IPA = "orthography2ipa"  # multilingual data-driven IPA (o2i lattice)
     AFRICA_G2P = "africa_g2p"  # rule-based G2P for 400+ African languages (native units or IPA)
     GHANA_G2P = "ghana_g2p"  # africa-g2p rules patched for 42 Ghanaian languages (native units or IPA)
+    MATOUB_KAB = "matoub_kab"  # kabyle — the rule set agbalu/Matoub-82M was fitted on

{scriptconv-0.0.4a31 → scriptconv-0.0.4a32}/scriptconv/phonemizers/registry.py

@@ -59,6 +59,7 @@ PHONEMIZER_REGISTRY: Dict[Phonemizer, Tuple[str, str, Optional[str]]] = {
     _P.ORTHOGRAPHY2IPA: (f"{_BASE}.o2ipa", "Orthography2IPAPhonemizer", "o2i"),
     # vendored (not an optional extra) -- see _vendored/africa_g2p
     _P.AFRICA_G2P: (f"{_BASE}.africa", "AfricaG2PPhonemizer", None),
+    _P.MATOUB_KAB: (f"{_BASE}.ber", "MatoubKabylePhonemizer", None),
     _P.GHANA_G2P: (f"{_BASE}.ghana", "GhanaG2PPhonemizer", "ghana"),
     # ported in the CJK/AR stage; registered here so no member is silently
     # unmapped — resolution raises ImportError naming the pending extra

{scriptconv-0.0.4a31 → scriptconv-0.0.4a32}/scriptconv/version.py

@@ -2,7 +2,7 @@
 VERSION_MAJOR = 0
 VERSION_MINOR = 0
 VERSION_BUILD = 4
-VERSION_ALPHA = 31
+VERSION_ALPHA = 32
 # END_VERSION_BLOCK
 
 VERSION_STR = f"{VERSION_MAJOR}.{VERSION_MINOR}.{VERSION_BUILD}"

{scriptconv-0.0.4a31 → scriptconv-0.0.4a32/scriptconv.egg-info}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: scriptconv
-Version: 0.0.4a31
+Version: 0.0.4a32
 Summary: Zero-dependency script & phoneme-notation core — ISO-15924 detection & metadata, IPA↔ARPABET/X-SAMPA/Lexique/Kirshenbaum/Cotovía/RFE, Buckwalter↔Arabic, Hangul→jamo, kana
 License: Apache-2.0
 Project-URL: Homepage, https://github.com/TigreGotico/scriptconv

{scriptconv-0.0.4a31 → scriptconv-0.0.4a32}/scriptconv.egg-info/SOURCES.txt

@@ -25,6 +25,7 @@ scriptconv/phonemizers/__init__.py
 scriptconv/phonemizers/africa.py
 scriptconv/phonemizers/ar.py
 scriptconv/phonemizers/base.py
+scriptconv/phonemizers/ber.py
 scriptconv/phonemizers/en.py
 scriptconv/phonemizers/enums.py
 scriptconv/phonemizers/eu.py
@@ -494,6 +495,7 @@ scriptconv/phonemizers/_vendored/mantoq/buck/__init__.py
 scriptconv/phonemizers/_vendored/mantoq/buck/phonetise_buckwalter.py
 scriptconv/phonemizers/_vendored/mantoq/buck/symbols.py
 scriptconv/phonemizers/_vendored/mantoq/buck/tokenization.py
+scriptconv/phonemizers/_vendored/matoub_kab/__init__.py
 tests/test_arpa_stress.py
 tests/test_cangjie.py
 tests/test_cli.py
@@ -511,6 +513,7 @@ tests/test_phonemizers_cjk_ar.py
 tests/test_phonemizers_friendly_import_errors.py
 tests/test_phonemizers_ghana.py
 tests/test_phonemizers_halabi_iqra.py
+tests/test_phonemizers_matoub_kab.py
 tests/test_phonemizers_phonetisaurus.py
 tests/test_phonemizers_pt_tugaphone.py
 tests/test_phonemizers_ru.py

{scriptconv-0.0.4a31 → scriptconv-0.0.4a32}/tests/test_phonemizers_base.py

@@ -26,7 +26,8 @@ class TestEnums(unittest.TestCase):
         self.assertEqual(Alphabet.XSAMPA.value, "x-sampa")
         self.assertEqual(Alphabet.VOSK.value, "vosk")
         self.assertEqual(Phonemizer.GHANA_G2P.value, "ghana_g2p")
-        self.assertEqual(len(list(Phonemizer)), 47)  # +HALABI, +IQRA, +GHANA_G2P, +PHONETISAURUS
+        self.assertEqual(Phonemizer.MATOUB_KAB.value, "matoub_kab")
+        self.assertEqual(len(list(Phonemizer)), 48)  # +HALABI, +IQRA, +GHANA_G2P, +PHONETISAURUS, +MATOUB_KAB
         self.assertEqual(len(list(Alphabet)), 23)  # incl. HALABI, VOSK, AFRICA_G2P
 
 

scriptconv-0.0.4a32/tests/test_phonemizers_matoub_kab.py

@@ -0,0 +1,137 @@
+"""The Matoub Kabyle route: it phonemizes, and it is not the africa-g2p route."""
+import ast
+import pathlib
+import sys
+import tempfile
+import unittest
+
+from scriptconv.phonemizers import ber as _ber_module
+from scriptconv.phonemizers._vendored import matoub_kab as _vendored_module
+from scriptconv.phonemizers.ber import MatoubKabylePhonemizer
+from scriptconv.phonemizers.enums import Alphabet, Phonemizer
+from scriptconv.phonemizers.registry import PHONEMIZER_REGISTRY, get_phonemizer
+
+_VENDORED = pathlib.Path(_vendored_module.__file__)
+_BER = pathlib.Path(_ber_module.__file__)
+
+
+def _import_roots(path):
+    """Root module of every import statement in *path*.
+
+    Relative imports are skipped: they cannot reach outside the package.
+    """
+    roots = set()
+    for node in ast.walk(ast.parse(path.read_text(encoding="utf-8"))):
+        if isinstance(node, ast.Import):
+            roots.update(alias.name.split(".")[0] for alias in node.names)
+        elif isinstance(node, ast.ImportFrom) and not node.level and node.module:
+            roots.add(node.module.split(".")[0])
+    return roots
+
+
+def _tmp_module(source):
+    handle = tempfile.NamedTemporaryFile("w", suffix=".py", delete=False,
+                                         encoding="utf-8")
+    with handle:
+        handle.write(source)
+    return pathlib.Path(handle.name)
+
+
+# Measured against the reference rules in agbalu/Matoub-82M
+# tokenization_matoub.py at revision 3d00056f, on 2026-09-25.
+REFERENCE = {
+    "taddart": "θædːærθ",
+    "ameqqran": "æməqːræn",
+    "azul fell awen": "æzul fəlː æwən",
+    "tamurt n leqbayel": "θæmurθ n ləqβæjəl",
+    "aqcic yecca aɣrum": "ɑqʃiʃ jəʃːæ ɑʁrum",
+}
+
+
+class TestMatoubKabyle(unittest.TestCase):
+    def setUp(self):
+        self.p = MatoubKabylePhonemizer()
+
+    def test_phonemizes_real_words(self):
+        for text, expected in REFERENCE.items():
+            with self.subTest(text=text):
+                self.assertEqual(self.p.phonemize_string(text, "kab"), expected)
+
+    def test_the_rules_that_make_this_route_its_own(self):
+        # Each assertion is one rule africa-g2p does not apply. If these ever
+        # start matching africa-g2p, the two routes have been conflated.
+        # spirantisation: t -> θ, b -> β, d -> ð
+        self.assertEqual(self.p.phonemize_string("tamurt", "kab"), "θæmurθ")
+        # gemination is length on the restored stop, not a doubled symbol
+        self.assertEqual(self.p.phonemize_string("taddart", "kab"), "θædːærθ")
+        self.assertNotIn("ðð", self.p.phonemize_string("taddart", "kab"))
+        # /a/ backs to ɑ beside a backing consonant, and stays æ otherwise
+        self.assertTrue(self.p.phonemize_string("aqcic", "kab").startswith("ɑ"))
+        self.assertTrue(self.p.phonemize_string("azul", "kab").startswith("æ"))
+
+    def test_registry_entry_resolves(self):
+        self.assertIn(Phonemizer.MATOUB_KAB, PHONEMIZER_REGISTRY)
+        module, name, extra = PHONEMIZER_REGISTRY[Phonemizer.MATOUB_KAB]
+        self.assertEqual(name, "MatoubKabylePhonemizer")
+        # no extra: the rules are vendored and need only the standard library
+        self.assertIsNone(extra)
+        self.assertIsInstance(get_phonemizer(Phonemizer.MATOUB_KAB),
+                              MatoubKabylePhonemizer)
+
+    def test_emits_ipa_and_chunks(self):
+        self.assertEqual(self.p.alphabet, Alphabet.IPA)
+        self.assertEqual(self.p.phonemize("azul", "kab"), [["æ", "z", "u", "l"]])
+
+    def test_only_kabyle(self):
+        self.assertEqual(MatoubKabylePhonemizer.supported_langs(), ["kab"])
+        for other in ("eng", "fra", "ber"):
+            with self.subTest(lang=other):
+                with self.assertRaises(ValueError):
+                    self.p.phonemize_string("azul", other)
+
+    def test_an_unknown_character_raises_and_is_never_dropped(self):
+        # The upstream file records that silent deletion cost three Kabyle
+        # consonants once. A symbol with no rule must be reported.
+        with self.assertRaises(ValueError):
+            self.p.phonemize_string("aβc", "kab")
+
+    def test_the_vendored_rules_need_no_third_party_import(self):
+        # Read the property off the source, not off sys.modules. An earlier
+        # form of this test asserted that "transformers" and "torch" were
+        # absent from sys.modules; neither is installed, so the assertion was
+        # true before the module was imported at all, and adding
+        # `import packaging` to the vendored file left it passing.
+        #
+        # The property is load-bearing: it is why the registry row carries
+        # extra=None, and why the rules are vendored instead of depended on. A
+        # convenience import added later would keep get_phonemizer working on
+        # the machine that added it and fail as an ImportError on a user
+        # install.
+        for path in (_VENDORED, _BER):
+            with self.subTest(path=path.name):
+                for root in _import_roots(path):
+                    self.assertTrue(
+                        root in sys.stdlib_module_names or root == "scriptconv",
+                        f"{path.name} imports {root!r}, which is neither the "
+                        f"standard library nor scriptconv itself; the registry "
+                        f"row advertises this backend as needing no extra")
+
+    def test_that_import_check_fails_on_a_planted_third_party_import(self):
+        # The control. Without it the check above could be vacuous for a new
+        # reason: a parser that collects nothing also reports nothing.
+        planted = _tmp_module("import packaging\nimport re\n")
+        self.assertEqual(_import_roots(planted), {"packaging", "re"})
+        self.assertNotIn("packaging", sys.stdlib_module_names)
+        # and a clean module yields only allowed roots
+        clean = _tmp_module("import re\nfrom typing import Final\n")
+        self.assertEqual(_import_roots(clean), {"re", "typing"})
+
+    def test_the_import_reader_sees_the_real_files(self):
+        # A reader that silently found no imports would make the check above
+        # pass for the wrong reason, so pin what it actually collects.
+        self.assertIn("re", _import_roots(_VENDORED))
+        self.assertIn("scriptconv", _import_roots(_BER))
+
+
+if __name__ == "__main__":
+    unittest.main()