scriptconv 0.0.3a12__tar.gz → 0.0.3a13__tar.gz

{scriptconv-0.0.3a12 → scriptconv-0.0.3a13}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: scriptconv
-Version: 0.0.3a12
+Version: 0.0.3a13
 Summary: Zero-dependency script & phoneme-notation core — ISO-15924 detection & metadata, IPA↔ARPABET/X-SAMPA/Lexique/Kirshenbaum/Cotovía/RFE, Buckwalter↔Arabic, Hangul→jamo, kana
 License: Apache-2.0
 Project-URL: Homepage, https://github.com/TigreGotico/scriptconv
@@ -78,6 +78,33 @@ Requires-Dist: viphoneme; extra == "vi"
 Provides-Extra: tashkeel
 Requires-Dist: scriptconv[phonemizers]; extra == "tashkeel"
 Requires-Dist: text2tashkeel; extra == "tashkeel"
+Provides-Extra: ja-phonemizers
+Requires-Dist: scriptconv[phonemizers]; extra == "ja-phonemizers"
+Requires-Dist: pyopenjtalk; extra == "ja-phonemizers"
+Requires-Dist: cutlet; extra == "ja-phonemizers"
+Requires-Dist: pykakasi<3,>=2.3; extra == "ja-phonemizers"
+Provides-Extra: ko
+Requires-Dist: scriptconv[phonemizers]; extra == "ko"
+Requires-Dist: g2pk; extra == "ko"
+Requires-Dist: regex; extra == "ko"
+Provides-Extra: zh-phonemizers
+Requires-Dist: scriptconv[phonemizers]; extra == "zh-phonemizers"
+Requires-Dist: jieba; extra == "zh-phonemizers"
+Requires-Dist: pypinyin<1,>=0.50; extra == "zh-phonemizers"
+Requires-Dist: xpinyin; extra == "zh-phonemizers"
+Requires-Dist: g2pM; extra == "zh-phonemizers"
+Requires-Dist: pinyin_to_ipa; extra == "zh-phonemizers"
+Provides-Extra: en-phonemizers
+Requires-Dist: scriptconv[phonemizers]; extra == "en-phonemizers"
+Requires-Dist: g2p_en; extra == "en-phonemizers"
+Requires-Dist: openphonemizer; extra == "en-phonemizers"
+Requires-Dist: deep-phonemizer; extra == "en-phonemizers"
+Provides-Extra: ar-phonemizers
+Requires-Dist: scriptconv[phonemizers]; extra == "ar-phonemizers"
+Requires-Dist: arbtok; extra == "ar-phonemizers"
+Provides-Extra: shami
+Requires-Dist: scriptconv[phonemizers]; extra == "shami"
+Requires-Dist: regex; extra == "shami"
 Provides-Extra: test
 Requires-Dist: pytest; extra == "test"
 Requires-Dist: pytest-timeout; extra == "test"
@@ -85,6 +112,7 @@ Requires-Dist: pykakasi<3,>=2.3; extra == "test"
 Requires-Dist: pypinyin<1,>=0.50; extra == "test"
 Requires-Dist: quebra-frases; extra == "test"
 Requires-Dist: langcodes; extra == "test"
+Requires-Dist: regex; extra == "test"
 Dynamic: license-file
 
 # scriptconv

{scriptconv-0.0.3a12 → scriptconv-0.0.3a13}/pyproject.toml

@@ -53,7 +53,13 @@ he = ["scriptconv[phonemizers]", "phonikud-onnx"]
 fa = ["scriptconv[phonemizers]", "persian_phonemizer"]
 vi = ["scriptconv[phonemizers]", "viphoneme"]
 tashkeel = ["scriptconv[phonemizers]", "text2tashkeel"]
-test = ["pytest", "pytest-timeout", "pykakasi>=2.3,<3", "pypinyin>=0.50,<1", "quebra-frases", "langcodes"]
+ja-phonemizers = ["scriptconv[phonemizers]", "pyopenjtalk", "cutlet", "pykakasi>=2.3,<3"]
+ko = ["scriptconv[phonemizers]", "g2pk", "regex"]
+zh-phonemizers = ["scriptconv[phonemizers]", "jieba", "pypinyin>=0.50,<1", "xpinyin", "g2pM", "pinyin_to_ipa"]
+en-phonemizers = ["scriptconv[phonemizers]", "g2p_en", "openphonemizer", "deep-phonemizer"]
+ar-phonemizers = ["scriptconv[phonemizers]", "arbtok"]
+shami = ["scriptconv[phonemizers]", "regex"]
+test = ["pytest", "pytest-timeout", "pykakasi>=2.3,<3", "pypinyin>=0.50,<1", "quebra-frases", "langcodes", "regex"]
 
 [project.urls]
 Homepage = "https://github.com/TigreGotico/scriptconv"
@@ -65,6 +71,7 @@ include = ["scriptconv*"]
 [tool.setuptools.package-data]
 scriptconv = ["py.typed"]
 "scriptconv.data" = ["*.tsv.gz"]
+"scriptconv.phonemizers._thirdparty" = ["ko_tables/*.csv", "ko_tables/*.tsv"]
 
 [tool.pytest.ini_options]
 testpaths = ["tests"]

scriptconv-0.0.3a13/scriptconv/phonemizers/_thirdparty/__init__.py

@@ -0,0 +1,7 @@
+"""Vendored first-party-maintained G2P implementations.
+
+Each module carries its upstream provenance and license in its header; only
+permissively-licensed code lives here (MIT/Apache).  GPL or NonCommercial
+upstreams (KoG2P, mantoq's pyarabic/phonetise) are NOT vendored — their
+wrapper classes raise with an explanation instead.
+"""

scriptconv-0.0.3a13/scriptconv/phonemizers/_thirdparty/bw2ipa.py

@@ -0,0 +1,207 @@
+# translate.py
+# Deterministic Mantoq → IPA converter
+# Assumes the input is already tokenized using the Mantoq inventory.
+
+# ---------------------------------------------------------------------------
+# Token → IPA maps
+# ---------------------------------------------------------------------------
+
+CONSONANTS = {
+    "b":  "b",
+    "t":  "t",
+    "^":  "θ",
+    "j":  "d͡ʒ",
+    "H":  "ħ",
+    "x":  "x",
+    "d":  "d",
+    "*":  "ð",
+    "r":  "r",
+    "z":  "z",
+    "s":  "s",
+    "$":  "ʃ",
+    "S":  "sˤ",
+    "D":  "dˤ",
+    "T":  "tˤ",
+    "Z":  "ðˤ",
+    "E":  "ʕ",
+    "g":  "ɣ",
+    "f":  "f",
+    "q":  "q",
+    "k":  "k",
+    "l":  "l",
+    "m":  "m",
+    "n":  "n",
+    "h":  "h",
+    "w":  "w",
+    "y":  "j",
+    "v":  "v"
+}
+
+VOWELS = {
+    "a":    "a",
+    "aa":   "aː",
+    "aaaa": "aːː",
+    "i":    "i",
+    "ii":   "iː",
+    "u":    "u",
+    "uu":   "uː",
+}
+
+
+# Punctuation is passed through unchanged:
+PUNCTUATION = set(list(".,;:!?()[]{}\"'"))
+
+# ---------------------------------------------------------------------------
+# Tokenizer
+# ---------------------------------------------------------------------------
+
+def tokenize_mantoq(text):
+    """
+    Tokenize Mantoq string deterministically.
+
+    Priority:
+        1. _dbl_
+        2. aaaa
+        3. ii / uu / aa
+        4. single char tokens
+        5. punctuation and raw chars are passed through
+    """
+    tokens = []
+    i = 0
+    L = len(text)
+
+    while i < L:
+
+        # doubling marker
+        if text.startswith("_dbl_", i):
+            tokens.append("_dbl_")
+            i += 5
+            continue
+
+        # word separator
+        if text.startswith("_+_", i):
+            tokens.append("_+_")
+            i += 3
+            continue
+
+        # longest vowel first
+        if text.startswith("aaaa", i):
+            tokens.append("aaaa")
+            i += 4
+            continue
+
+        if text.startswith("aa", i):
+            tokens.append("aa")
+            i += 2
+            continue
+
+        if text.startswith("ii", i):
+            tokens.append("ii")
+            i += 2
+            continue
+
+        if text.startswith("uu", i):
+            tokens.append("uu")
+            i += 2
+            continue
+
+        # single-character consonant or vowel
+        ch = text[i]
+
+        if ch in CONSONANTS or ch in VOWELS:
+            tokens.append(ch)
+            i += 1
+            continue
+        if ch == "<":
+            tokens.append("ʔ")
+            i += 1
+            continue
+
+        # punctuation
+        if ch in PUNCTUATION:
+            tokens.append(ch)
+            i += 1
+            continue
+
+        # fallback: pass through unknown characters
+        tokens.append(ch)
+        i += 1
+
+    return tokens
+
+# ---------------------------------------------------------------------------
+# IPA Assembly
+# ---------------------------------------------------------------------------
+
+def apply_doubling(prev_token, prev_ipa):
+    """
+    Mantoq doubling rule:
+       - If previous token is a vowel token: lengthen it.
+       - If previous token is a consonant: mark gemination using ː.
+    """
+    if prev_token in VOWELS:
+        # ensure single long marker; long tokens already contain ː
+        if prev_ipa.endswith("ː"):
+            return prev_ipa + "ː"
+        return prev_ipa + "ː"
+
+    if prev_token in CONSONANTS:
+        # consonant gemination: use length mark, not duplication
+        if prev_ipa.endswith("ː"):
+            return prev_ipa  # already geminated
+        return prev_ipa + "ː"
+
+    return prev_ipa
+
+
+def mantoq_to_ipa(text):
+    tokens = tokenize_mantoq(text)
+
+    ipa_out = []
+    last_token = None
+    last_ipa = None
+
+    for tok in tokens:
+
+        # doubling applies to the previous symbol
+        if tok == "_dbl_":
+            if last_token is None:
+                continue
+            new_ipa = apply_doubling(last_token, last_ipa)
+            ipa_out[-1] = new_ipa
+            last_ipa = new_ipa
+            continue
+
+        # explicit word separation
+        if tok == "_+_":
+            ipa_out.append(" ")
+            last_token = tok
+            last_ipa = " "
+            continue
+
+        # vowels
+        if tok in VOWELS:
+            ipa_val = VOWELS[tok]
+            ipa_out.append(ipa_val)
+            last_token = tok
+            last_ipa = ipa_val
+            continue
+
+        # consonants
+        if tok in CONSONANTS:
+            ipa_val = CONSONANTS[tok]
+            ipa_out.append(ipa_val)
+            last_token = tok
+            last_ipa = ipa_val
+            continue
+
+        # punctuation and fallthrough
+        ipa_out.append(tok)
+        last_token = tok
+        last_ipa = tok
+
+    return "".join(ipa_out)
+
+
+# backwards compat alias
+bw2ipa = mantoq_to_ipa