scriptconv 0.0.3a11__tar.gz → 0.0.3a13__tar.gz

{scriptconv-0.0.3a11 → scriptconv-0.0.3a13}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: scriptconv
-Version: 0.0.3a11
+Version: 0.0.3a13
 Summary: Zero-dependency script & phoneme-notation core — ISO-15924 detection & metadata, IPA↔ARPABET/X-SAMPA/Lexique/Kirshenbaum/Cotovía/RFE, Buckwalter↔Arabic, Hangul→jamo, kana
 License: Apache-2.0
 Project-URL: Homepage, https://github.com/TigreGotico/scriptconv
@@ -24,11 +24,95 @@ Provides-Extra: ja
 Requires-Dist: pykakasi<3,>=2.3; extra == "ja"
 Provides-Extra: zh
 Requires-Dist: pypinyin<1,>=0.50; extra == "zh"
+Provides-Extra: phonemizers
+Requires-Dist: quebra-frases; extra == "phonemizers"
+Requires-Dist: langcodes; extra == "phonemizers"
+Provides-Extra: espeak
+Requires-Dist: scriptconv[phonemizers]; extra == "espeak"
+Requires-Dist: espyak; extra == "espeak"
+Provides-Extra: gruut
+Requires-Dist: scriptconv[phonemizers]; extra == "gruut"
+Requires-Dist: gruut; extra == "gruut"
+Provides-Extra: goruut
+Requires-Dist: scriptconv[phonemizers]; extra == "goruut"
+Requires-Dist: pygoruut; extra == "goruut"
+Provides-Extra: epitran
+Requires-Dist: scriptconv[phonemizers]; extra == "epitran"
+Requires-Dist: epitran; extra == "epitran"
+Provides-Extra: transphone
+Requires-Dist: scriptconv[phonemizers]; extra == "transphone"
+Requires-Dist: transphone; extra == "transphone"
+Provides-Extra: misaki
+Requires-Dist: scriptconv[phonemizers]; extra == "misaki"
+Requires-Dist: misaki; extra == "misaki"
+Provides-Extra: byt5
+Requires-Dist: scriptconv[phonemizers]; extra == "byt5"
+Requires-Dist: onnxruntime; extra == "byt5"
+Requires-Dist: numpy; extra == "byt5"
+Provides-Extra: o2i
+Requires-Dist: scriptconv[phonemizers]; extra == "o2i"
+Requires-Dist: orthography2ipa; extra == "o2i"
+Provides-Extra: eu
+Requires-Dist: scriptconv[phonemizers]; extra == "eu"
+Requires-Dist: euskaphone; extra == "eu"
+Requires-Dist: ahotts_g2p; extra == "eu"
+Provides-Extra: pt-phonemizers
+Requires-Dist: scriptconv[phonemizers]; extra == "pt-phonemizers"
+Requires-Dist: tugaphone; extra == "pt-phonemizers"
+Requires-Dist: g2p_barranquenho; extra == "pt-phonemizers"
+Provides-Extra: mwl
+Requires-Dist: scriptconv[phonemizers]; extra == "mwl"
+Requires-Dist: mwl_phonemizer; extra == "mwl"
+Provides-Extra: gl
+Requires-Dist: scriptconv[phonemizers]; extra == "gl"
+Requires-Dist: pycotovia; extra == "gl"
+Provides-Extra: he
+Requires-Dist: scriptconv[phonemizers]; extra == "he"
+Requires-Dist: phonikud-onnx; extra == "he"
+Provides-Extra: fa
+Requires-Dist: scriptconv[phonemizers]; extra == "fa"
+Requires-Dist: persian_phonemizer; extra == "fa"
+Provides-Extra: vi
+Requires-Dist: scriptconv[phonemizers]; extra == "vi"
+Requires-Dist: viphoneme; extra == "vi"
+Provides-Extra: tashkeel
+Requires-Dist: scriptconv[phonemizers]; extra == "tashkeel"
+Requires-Dist: text2tashkeel; extra == "tashkeel"
+Provides-Extra: ja-phonemizers
+Requires-Dist: scriptconv[phonemizers]; extra == "ja-phonemizers"
+Requires-Dist: pyopenjtalk; extra == "ja-phonemizers"
+Requires-Dist: cutlet; extra == "ja-phonemizers"
+Requires-Dist: pykakasi<3,>=2.3; extra == "ja-phonemizers"
+Provides-Extra: ko
+Requires-Dist: scriptconv[phonemizers]; extra == "ko"
+Requires-Dist: g2pk; extra == "ko"
+Requires-Dist: regex; extra == "ko"
+Provides-Extra: zh-phonemizers
+Requires-Dist: scriptconv[phonemizers]; extra == "zh-phonemizers"
+Requires-Dist: jieba; extra == "zh-phonemizers"
+Requires-Dist: pypinyin<1,>=0.50; extra == "zh-phonemizers"
+Requires-Dist: xpinyin; extra == "zh-phonemizers"
+Requires-Dist: g2pM; extra == "zh-phonemizers"
+Requires-Dist: pinyin_to_ipa; extra == "zh-phonemizers"
+Provides-Extra: en-phonemizers
+Requires-Dist: scriptconv[phonemizers]; extra == "en-phonemizers"
+Requires-Dist: g2p_en; extra == "en-phonemizers"
+Requires-Dist: openphonemizer; extra == "en-phonemizers"
+Requires-Dist: deep-phonemizer; extra == "en-phonemizers"
+Provides-Extra: ar-phonemizers
+Requires-Dist: scriptconv[phonemizers]; extra == "ar-phonemizers"
+Requires-Dist: arbtok; extra == "ar-phonemizers"
+Provides-Extra: shami
+Requires-Dist: scriptconv[phonemizers]; extra == "shami"
+Requires-Dist: regex; extra == "shami"
 Provides-Extra: test
 Requires-Dist: pytest; extra == "test"
 Requires-Dist: pytest-timeout; extra == "test"
 Requires-Dist: pykakasi<3,>=2.3; extra == "test"
 Requires-Dist: pypinyin<1,>=0.50; extra == "test"
+Requires-Dist: quebra-frases; extra == "test"
+Requires-Dist: langcodes; extra == "test"
+Requires-Dist: regex; extra == "test"
 Dynamic: license-file
 
 # scriptconv

{scriptconv-0.0.3a11 → scriptconv-0.0.3a13}/pyproject.toml

@@ -35,7 +35,31 @@ dependencies = {file = ["requirements.txt"]}
 [project.optional-dependencies]
 ja = ["pykakasi>=2.3,<3"]
 zh = ["pypinyin>=0.50,<1"]
-test = ["pytest", "pytest-timeout", "pykakasi>=2.3,<3", "pypinyin>=0.50,<1"]
+# --- phonemizer wrappers (scriptconv.phonemizers) ---
+phonemizers = ["quebra-frases", "langcodes"]
+espeak = ["scriptconv[phonemizers]", "espyak"]
+gruut = ["scriptconv[phonemizers]", "gruut"]
+goruut = ["scriptconv[phonemizers]", "pygoruut"]
+epitran = ["scriptconv[phonemizers]", "epitran"]
+transphone = ["scriptconv[phonemizers]", "transphone"]
+misaki = ["scriptconv[phonemizers]", "misaki"]
+byt5 = ["scriptconv[phonemizers]", "onnxruntime", "numpy"]
+o2i = ["scriptconv[phonemizers]", "orthography2ipa"]
+eu = ["scriptconv[phonemizers]", "euskaphone", "ahotts_g2p"]
+pt-phonemizers = ["scriptconv[phonemizers]", "tugaphone", "g2p_barranquenho"]
+mwl = ["scriptconv[phonemizers]", "mwl_phonemizer"]
+gl = ["scriptconv[phonemizers]", "pycotovia"]
+he = ["scriptconv[phonemizers]", "phonikud-onnx"]
+fa = ["scriptconv[phonemizers]", "persian_phonemizer"]
+vi = ["scriptconv[phonemizers]", "viphoneme"]
+tashkeel = ["scriptconv[phonemizers]", "text2tashkeel"]
+ja-phonemizers = ["scriptconv[phonemizers]", "pyopenjtalk", "cutlet", "pykakasi>=2.3,<3"]
+ko = ["scriptconv[phonemizers]", "g2pk", "regex"]
+zh-phonemizers = ["scriptconv[phonemizers]", "jieba", "pypinyin>=0.50,<1", "xpinyin", "g2pM", "pinyin_to_ipa"]
+en-phonemizers = ["scriptconv[phonemizers]", "g2p_en", "openphonemizer", "deep-phonemizer"]
+ar-phonemizers = ["scriptconv[phonemizers]", "arbtok"]
+shami = ["scriptconv[phonemizers]", "regex"]
+test = ["pytest", "pytest-timeout", "pykakasi>=2.3,<3", "pypinyin>=0.50,<1", "quebra-frases", "langcodes", "regex"]
 
 [project.urls]
 Homepage = "https://github.com/TigreGotico/scriptconv"
@@ -47,6 +71,7 @@ include = ["scriptconv*"]
 [tool.setuptools.package-data]
 scriptconv = ["py.typed"]
 "scriptconv.data" = ["*.tsv.gz"]
+"scriptconv.phonemizers._thirdparty" = ["ko_tables/*.csv", "ko_tables/*.tsv"]
 
 [tool.pytest.ini_options]
 testpaths = ["tests"]

scriptconv-0.0.3a13/scriptconv/phonemizers/__init__.py

@@ -0,0 +1,40 @@
+"""Phonemizer wrappers — see :mod:`scriptconv.phonemizers.base` for the thesis.
+
+Public surface: the :class:`BasePhonemizer` contract, the ``Alphabet``/
+``Phonemizer`` enums, the lazy registry (``get_phonemizer``,
+``phonemizer_for_lang``), and the per-language wrapper modules.  Backing
+engines are optional extras; a missing one raises :class:`ImportError`
+naming the extra to install.
+"""
+from scriptconv.phonemizers.enums import Alphabet, Phonemizer
+from scriptconv.phonemizers.base import (
+    BasePhonemizer,
+    GraphemePhonemizer,
+    UnicodeCodepointPhonemizer,
+    TextChunks,
+    RawPhonemizedChunks,
+    PhonemizedChunks,
+)
+from scriptconv.phonemizers.registry import (
+    PHONEMIZER_REGISTRY,
+    LANG_DEFAULTS,
+    get_phonemizer,
+    get_phonemizer_class,
+    phonemizer_for_lang,
+)
+
+__all__ = [
+    "Alphabet",
+    "Phonemizer",
+    "BasePhonemizer",
+    "GraphemePhonemizer",
+    "UnicodeCodepointPhonemizer",
+    "TextChunks",
+    "RawPhonemizedChunks",
+    "PhonemizedChunks",
+    "PHONEMIZER_REGISTRY",
+    "LANG_DEFAULTS",
+    "get_phonemizer",
+    "get_phonemizer_class",
+    "phonemizer_for_lang",
+]

scriptconv-0.0.3a13/scriptconv/phonemizers/_thirdparty/__init__.py

@@ -0,0 +1,7 @@
+"""Vendored first-party-maintained G2P implementations.
+
+Each module carries its upstream provenance and license in its header; only
+permissively-licensed code lives here (MIT/Apache).  GPL or NonCommercial
+upstreams (KoG2P, mantoq's pyarabic/phonetise) are NOT vendored — their
+wrapper classes raise with an explanation instead.
+"""

scriptconv-0.0.3a13/scriptconv/phonemizers/_thirdparty/bw2ipa.py

@@ -0,0 +1,207 @@
+# translate.py
+# Deterministic Mantoq → IPA converter
+# Assumes the input is already tokenized using the Mantoq inventory.
+
+# ---------------------------------------------------------------------------
+# Token → IPA maps
+# ---------------------------------------------------------------------------
+
+CONSONANTS = {
+    "b":  "b",
+    "t":  "t",
+    "^":  "θ",
+    "j":  "d͡ʒ",
+    "H":  "ħ",
+    "x":  "x",
+    "d":  "d",
+    "*":  "ð",
+    "r":  "r",
+    "z":  "z",
+    "s":  "s",
+    "$":  "ʃ",
+    "S":  "sˤ",
+    "D":  "dˤ",
+    "T":  "tˤ",
+    "Z":  "ðˤ",
+    "E":  "ʕ",
+    "g":  "ɣ",
+    "f":  "f",
+    "q":  "q",
+    "k":  "k",
+    "l":  "l",
+    "m":  "m",
+    "n":  "n",
+    "h":  "h",
+    "w":  "w",
+    "y":  "j",
+    "v":  "v"
+}
+
+VOWELS = {
+    "a":    "a",
+    "aa":   "aː",
+    "aaaa": "aːː",
+    "i":    "i",
+    "ii":   "iː",
+    "u":    "u",
+    "uu":   "uː",
+}
+
+
+# Punctuation is passed through unchanged:
+PUNCTUATION = set(list(".,;:!?()[]{}\"'"))
+
+# ---------------------------------------------------------------------------
+# Tokenizer
+# ---------------------------------------------------------------------------
+
+def tokenize_mantoq(text):
+    """
+    Tokenize Mantoq string deterministically.
+
+    Priority:
+        1. _dbl_
+        2. aaaa
+        3. ii / uu / aa
+        4. single char tokens
+        5. punctuation and raw chars are passed through
+    """
+    tokens = []
+    i = 0
+    L = len(text)
+
+    while i < L:
+
+        # doubling marker
+        if text.startswith("_dbl_", i):
+            tokens.append("_dbl_")
+            i += 5
+            continue
+
+        # word separator
+        if text.startswith("_+_", i):
+            tokens.append("_+_")
+            i += 3
+            continue
+
+        # longest vowel first
+        if text.startswith("aaaa", i):
+            tokens.append("aaaa")
+            i += 4
+            continue
+
+        if text.startswith("aa", i):
+            tokens.append("aa")
+            i += 2
+            continue
+
+        if text.startswith("ii", i):
+            tokens.append("ii")
+            i += 2
+            continue
+
+        if text.startswith("uu", i):
+            tokens.append("uu")
+            i += 2
+            continue
+
+        # single-character consonant or vowel
+        ch = text[i]
+
+        if ch in CONSONANTS or ch in VOWELS:
+            tokens.append(ch)
+            i += 1
+            continue
+        if ch == "<":
+            tokens.append("ʔ")
+            i += 1
+            continue
+
+        # punctuation
+        if ch in PUNCTUATION:
+            tokens.append(ch)
+            i += 1
+            continue
+
+        # fallback: pass through unknown characters
+        tokens.append(ch)
+        i += 1
+
+    return tokens
+
+# ---------------------------------------------------------------------------
+# IPA Assembly
+# ---------------------------------------------------------------------------
+
+def apply_doubling(prev_token, prev_ipa):
+    """
+    Mantoq doubling rule:
+       - If previous token is a vowel token: lengthen it.
+       - If previous token is a consonant: mark gemination using ː.
+    """
+    if prev_token in VOWELS:
+        # ensure single long marker; long tokens already contain ː
+        if prev_ipa.endswith("ː"):
+            return prev_ipa + "ː"
+        return prev_ipa + "ː"
+
+    if prev_token in CONSONANTS:
+        # consonant gemination: use length mark, not duplication
+        if prev_ipa.endswith("ː"):
+            return prev_ipa  # already geminated
+        return prev_ipa + "ː"
+
+    return prev_ipa
+
+
+def mantoq_to_ipa(text):
+    tokens = tokenize_mantoq(text)
+
+    ipa_out = []
+    last_token = None
+    last_ipa = None
+
+    for tok in tokens:
+
+        # doubling applies to the previous symbol
+        if tok == "_dbl_":
+            if last_token is None:
+                continue
+            new_ipa = apply_doubling(last_token, last_ipa)
+            ipa_out[-1] = new_ipa
+            last_ipa = new_ipa
+            continue
+
+        # explicit word separation
+        if tok == "_+_":
+            ipa_out.append(" ")
+            last_token = tok
+            last_ipa = " "
+            continue
+
+        # vowels
+        if tok in VOWELS:
+            ipa_val = VOWELS[tok]
+            ipa_out.append(ipa_val)
+            last_token = tok
+            last_ipa = ipa_val
+            continue
+
+        # consonants
+        if tok in CONSONANTS:
+            ipa_val = CONSONANTS[tok]
+            ipa_out.append(ipa_val)
+            last_token = tok
+            last_ipa = ipa_val
+            continue
+
+        # punctuation and fallthrough
+        ipa_out.append(tok)
+        last_token = tok
+        last_ipa = tok
+
+    return "".join(ipa_out)
+
+
+# backwards compat alias
+bw2ipa = mantoq_to_ipa