scriptconv 0.0.3a18__tar.gz → 0.0.3a20__tar.gz

{scriptconv-0.0.3a18 → scriptconv-0.0.3a20}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: scriptconv
-Version: 0.0.3a18
+Version: 0.0.3a20
 Summary: Zero-dependency script & phoneme-notation core — ISO-15924 detection & metadata, IPA↔ARPABET/X-SAMPA/Lexique/Kirshenbaum/Cotovía/RFE, Buckwalter↔Arabic, Hangul→jamo, kana
 License: Apache-2.0
 Project-URL: Homepage, https://github.com/TigreGotico/scriptconv

{scriptconv-0.0.3a18 → scriptconv-0.0.3a20}/scriptconv/notation.py

@@ -1034,7 +1034,8 @@ _MANTOQ_VOWELS: dict[str, str] = {
     "u": "u", "uu": "uː",
 }
 
-_MANTOQ_MULTI = ("_dbl_", "_+_", "aaaa", "aa", "ii", "uu")
+_MANTOQ_MULTI = ("_dbl_", "_+_", "_sil_", "_eos_", "_pad_",
+                 "aaaa", "aa", "ii", "uu")
 
 
 def _tokenize_mantoq(text: str) -> list[str]:
@@ -1052,7 +1053,7 @@ def _tokenize_mantoq(text: str) -> list[str]:
     return tokens
 
 
-def mantoq_to_ipa(mantoq: str, errors: str = "pass") -> str:
+def mantoq_to_ipa(mantoq, errors: str = "pass") -> str:
     """Convert a Mantoq phoneme string to IPA.
 
     ``_dbl_`` lengthens/geminates the preceding symbol (``ː``), ``_+_``
@@ -1065,16 +1066,33 @@ def mantoq_to_ipa(mantoq: str, errors: str = "pass") -> str:
     's a l aː m'
     >>> mantoq_to_ipa("b_dbl_a")
     'bːa'
+
+    The inventory's special tokens are honoured: ``_sil_`` becomes a space,
+    ``_eos_``/``_pad_`` are dropped.
     """
+    # a pre-tokenized sequence (as returned by the mantoq package's g2p)
+    # is consumed directly — joining tokens into a string is ambiguous
+    # ("a"+"aa" and "aa"+"a" both join to "aaa")
+    tokens = _tokenize_mantoq(mantoq) if isinstance(mantoq, str) else mantoq
     out: list[str] = []
     last_tok = None
-    for pos, tok in enumerate(_tokenize_mantoq(mantoq)):
+    for pos, tok in enumerate(tokens):
         if tok == "_dbl_":
-            # vowels lengthen cumulatively (aa + _dbl_ -> aːː); consonants
-            # geminate once
-            if out and last_tok is not None:
-                if last_tok in _MANTOQ_VOWELS or not out[-1].endswith("ː"):
-                    out[-1] += "ː"
+            # upstream emits _dbl_ only by splitting a doubled CONSONANT
+            # (mantoq buck/tokenization.py: `phon not in vowels and
+            # phon[0] == phon[1]`); geminate it with a length mark.  A
+            # vowel+_dbl_ sequence does not occur in genuine mantoq output
+            # and is left unchanged.
+            if out and last_tok in _MANTOQ_CONSONANTS \
+                    and not out[-1].endswith("ː"):
+                out[-1] += "ː"
+            continue
+        if tok == "_sil_":
+            out.append(" ")
+            last_tok = None
+            continue
+        if tok in ("_eos_", "_pad_"):
+            last_tok = None
             continue
         if tok == "_+_":
             out.append(" ")

{scriptconv-0.0.3a18 → scriptconv-0.0.3a20}/scriptconv/phonemizers/ar.py

@@ -5,21 +5,46 @@ from scriptconv.phonemizers.base import BasePhonemizer
 
 
 class MantoqPhonemizer(BasePhonemizer):
-    """mantoq-backed Arabic phonemizer — NOT distributable with scriptconv.
+    """Arabic phonemizer wrapping the external ``mantoq`` package.
 
-    mantoq's phonetise_buckwalter is CC BY-NC 4.0 (non-commercial) and its
-    vendored pyarabic is GPL — both incompatible with this library's
-    Apache-2.0 distribution, so the implementation is not vendored here.
-    Use :class:`ArbtokPhonemizer` (the Arabic default) instead.
+    mantoq (github.com/mush42/mantoq) bundles Nawar Halabi's Arabic-Phonetiser,
+    whose phonetisation core is CC BY-NC 4.0 — a license incompatible with
+    vendoring in this Apache-2.0 library, so the package is imported at
+    runtime and must be installed separately by a user who accepts its terms:
+
+        pip install git+https://github.com/mush42/mantoq
+
+    Kept for compatibility with published models trained on mantoq phoneme
+    sequences; the Arabic default is :class:`ArbtokPhonemizer`.  Output is
+    IPA (via :func:`scriptconv.notation.mantoq_to_ipa`) or the raw mantoq
+    inventory with ``alphabet=Alphabet.MANTOQ``.
     """
 
-    def __init__(self, *args, **kwargs):
-        raise ImportError(
-            "mantoq is CC BY-NC / GPL licensed and not vendored in "
-            "scriptconv — use ArbtokPhonemizer (the Arabic default)")
+    def __init__(self, alphabet: Alphabet = Alphabet.IPA):
+        if alphabet not in (Alphabet.IPA, Alphabet.MANTOQ):
+            raise ValueError("MantoqPhonemizer outputs IPA or MANTOQ")
+        super().__init__(alphabet)
+        try:
+            import mantoq
+        except ImportError:
+            raise ImportError(
+                "the mantoq package is not installed (and not vendored here: "
+                "its phonetisation core is CC BY-NC 4.0). Install it "
+                "separately, accepting its license terms: "
+                "pip install git+https://github.com/mush42/mantoq") from None
+        self._g2p = mantoq.g2p
+
+    @classmethod
+    def get_lang(cls, target_lang: str) -> str:
+        return cls.match_lang(target_lang, ["ar"])
 
     def phonemize_string(self, text: str, lang: str) -> str:
-        raise NotImplementedError
+        _, tokens = self._g2p(text)
+        if self.alphabet == Alphabet.MANTOQ:
+            return " ".join(tokens)
+        from scriptconv.notation import mantoq_to_ipa
+        return mantoq_to_ipa(tokens)
+
 
 
 class ArbtokPhonemizer(BasePhonemizer):

{scriptconv-0.0.3a18 → scriptconv-0.0.3a20}/scriptconv/phonemizers/ko.py

@@ -47,19 +47,30 @@ class G2PKPhonemizer(BasePhonemizer):
 
 
 class KoG2PPhonemizer(BasePhonemizer):
-    """KoG2P-backed Korean phonemizer — NOT distributable with scriptconv.
+    """Korean phonemizer wrapping an external ``kog2p`` distribution.
 
-    Upstream KoG2P (github.com/scarletcho/KoG2P) is GPL-3.0, incompatible
-    with this library's Apache-2.0 distribution, so the implementation is not
-    vendored here.  Use :class:`G2PKPhonemizer` or the hangul2ipa-backed
-    pipeline instead, or a GPL-compatible package that ships KoG2P itself.
+    Upstream KoG2P (github.com/scarletcho/KoG2P) is GPL-3.0 — a license
+    incompatible with vendoring in this Apache-2.0 library, so a ``kog2p``
+    module providing ``runKoG2P`` must be installed separately by a user who
+    accepts its terms.  :class:`G2PKPhonemizer` is the unencumbered
+    alternative.
     """
 
-    def __init__(self, *args, **kwargs):
-        raise ImportError(
-            "KoG2P is GPL-3.0 and not vendored in scriptconv — use "
-            "G2PKPhonemizer, or install a GPL-licensed distribution that "
-            "provides it")
+    def __init__(self, alphabet: Alphabet = Alphabet.IPA):
+        super().__init__(alphabet)
+        try:
+            from kog2p import runKoG2P
+        except ImportError:
+            raise ImportError(
+                "no kog2p module installed (KoG2P is GPL-3.0 and not "
+                "vendored here). Install a distribution providing it, "
+                "accepting its license, or use G2PKPhonemizer") from None
+        self.g2p = runKoG2P
+
+    @classmethod
+    def get_lang(cls, target_lang: str) -> str:
+        return cls.match_lang(target_lang, ["ko"])
 
     def phonemize_string(self, text: str, lang: str) -> str:
-        raise NotImplementedError
+        # contract: the installed module's runKoG2P(text) -> phoneme string
+        return self.g2p(text)

{scriptconv-0.0.3a18 → scriptconv-0.0.3a20}/scriptconv/version.py

@@ -2,7 +2,7 @@
 VERSION_MAJOR = 0
 VERSION_MINOR = 0
 VERSION_BUILD = 3
-VERSION_ALPHA = 18
+VERSION_ALPHA = 20
 # END_VERSION_BLOCK
 
 VERSION_STR = f"{VERSION_MAJOR}.{VERSION_MINOR}.{VERSION_BUILD}"

{scriptconv-0.0.3a18 → scriptconv-0.0.3a20}/scriptconv.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: scriptconv
-Version: 0.0.3a18
+Version: 0.0.3a20
 Summary: Zero-dependency script & phoneme-notation core — ISO-15924 detection & metadata, IPA↔ARPABET/X-SAMPA/Lexique/Kirshenbaum/Cotovía/RFE, Buckwalter↔Arabic, Hangul→jamo, kana
 License: Apache-2.0
 Project-URL: Homepage, https://github.com/TigreGotico/scriptconv

{scriptconv-0.0.3a18 → scriptconv-0.0.3a20}/tests/test_notation.py

@@ -1168,7 +1168,20 @@ def test_mantoq_theta_is_caret_not_shadda():
 
 def test_mantoq_gemination_marker():
     assert mantoq_to_ipa("b_dbl_a") == "bːa"
-    assert mantoq_to_ipa("aa_dbl_") == "aːː"
+
+
+def test_mantoq_dbl_after_vowel_is_inert():
+    # upstream only emits _dbl_ after consonants (tokenization.py splits
+    # doubled consonant phonemes); a vowel+_dbl_ input is left unchanged
+    assert mantoq_to_ipa("aa_dbl_") == "aː"
+
+
+def test_mantoq_special_tokens():
+    # inventory special tokens: _sil_ -> pause (space), _eos_/_pad_ dropped —
+    # and never shredded into letter phonemes
+    assert mantoq_to_ipa("m_sil_n") == "m n"
+    assert mantoq_to_ipa("m a_eos_") == "m a"
+    assert mantoq_to_ipa("_pad_b") == "b"
 
 
 def test_mantoq_word_separator_and_glottal():

{scriptconv-0.0.3a18 → scriptconv-0.0.3a20}/tests/test_phonemizers_cjk_ar.py

@@ -41,13 +41,13 @@ class TestShamiFrontend(unittest.TestCase):
 
 
 class TestLicensingStubs(unittest.TestCase):
-    def test_mantoq_stub_explains_licensing(self):
+    def test_mantoq_wrapper_requires_external_package(self):
         cls = get_phonemizer_class(Phonemizer.MANTOQ)
         with self.assertRaises(ImportError) as ctx:
             cls()
         self.assertIn("BY-NC", str(ctx.exception).replace("CC BY-NC", "BY-NC"))
 
-    def test_kog2p_stub_explains_licensing(self):
+    def test_kog2p_wrapper_requires_external_package(self):
         cls = get_phonemizer_class(Phonemizer.KOG2PK)
         with self.assertRaises(ImportError) as ctx:
             cls()
@@ -67,3 +67,20 @@ class TestLicensingStubs(unittest.TestCase):
 
 if __name__ == "__main__":
     unittest.main()
+
+
+class TestMantoqTokensToIpa(unittest.TestCase):
+    def test_pretokenized_sequence_accepted(self):
+        # the mantoq package's g2p returns a token LIST; joining is ambiguous
+        # so mantoq_to_ipa consumes sequences directly
+        from scriptconv.notation import mantoq_to_ipa
+        tokens = ["m", "a", "r", "H", "a", "b", "a", "n", "aa",
+                  "_+_", "b", "i", "l", "E", "aa", "l", "a", "m", "i"]
+        out = mantoq_to_ipa(tokens)
+        self.assertIn("ħ", out)
+        self.assertIn(" ", out)
+        self.assertNotIn("_", out)
+
+    def test_pretokenized_dbl_geminate(self):
+        from scriptconv.notation import mantoq_to_ipa
+        self.assertEqual(mantoq_to_ipa(["b", "_dbl_", "a"]), "bːa")