phoonnx 0.0.2a2__py3-none-any.whl → 0.1.0a3__py3-none-any.whl

phoonnx/thirdparty/bw2ipa.py

@@ -0,0 +1,66 @@
+# -*- coding: UTF8 -*-
+"""
+This script translates Buckwalter-transcribed Modern Standard Arabic to IPA.
+
+This relies on Mantoq tokenization that uses separate tokens for vowel length and consonant gemination.
+
+by Casimiro Ferreira with help of Gemini July 2025.
+"""
+
+import re
+
+# This dictionary maps a single Buckwalter character to its most common IPA equivalent.
+char_dict = {
+    'a': 'a', 'A': 'aː', 'b': 'b', 'c': 'x', 'd': 'd', 'D': 'dˤ', 'e': 'e', 'E': 'ʕ',
+    'f': 'f', 'g': 'ɣ', 'h': 'h', 'H': 'ħ', 'i': 'i', 'I': 'iː', 'j': 'ʒ', 'k': 'k',
+    'l': 'l', 'm': 'm', 'n': 'n', 'p': 'p', 'q': 'q', 'r': 'r', 'R': 'r', 's': 's',
+    'S': 'sˤ', 't': 't', 'T': 'tˤ', 'u': 'u', 'U': 'uː', 'v': 'v', 'w': 'w', 'x': 'x',
+    'y': 'j', 'z': 'z', 'Z': 'ðˤ', '\'': 'ʔ', '<': 'ʔ', 'o': 'o', '-': ' ',
+    '*': 'ð', '$': 'ʃ'
+}
+_vowels = {'a', 'i', 'u', 'aː', 'iː', 'uː'}
+
+
+def translate(buckwalter_text: str) -> str:
+    """
+    Translates a Buckwalter-transcribed string into an IPA string.
+
+    Args:
+        buckwalter_text (str): The Buckwalter string to translate.
+
+    Returns:
+        str: The translated IPA string.
+    """
+    ipa_list = []
+    i = 0
+    while i < len(buckwalter_text):
+        # Check for the longest token first. The new Mantoq tokenization
+        # seems to use a 5-character token.
+        token = buckwalter_text[i:i + 5]
+
+        # If the previous character was a vowel, we assume it's a long vowel
+        # marker (ː). Otherwise, we assume it's a geminated consonant.
+        if token == '_dbl_':
+            if ipa_list and ipa_list[-1] in _vowels:
+                ipa_list.append('ː')  # Add length marker for long vowels
+            elif ipa_list:
+                ipa_list.append(ipa_list[-1])  # Duplicate the consonant
+            i += 5
+            continue
+
+        # Check for multi-character mappings from char_dict
+        two_char_token = buckwalter_text[i:i + 2]
+        if two_char_token in char_dict:
+            ipa_list.append(char_dict[two_char_token])
+            i += 2
+            continue
+
+        # Handle single characters
+        single_char = buckwalter_text[i]
+        if single_char in char_dict:
+            ipa_list.append(char_dict[single_char])
+        else:
+            ipa_list.append(single_char)
+        i += 1
+
+    return ''.join(ipa_list)

phoonnx/thirdparty/hangul2ipa.py

@@ -1,3 +1,4 @@
+# taken from https://github.com/stannam/hangul_to_ipa
 import csv
 import math
 import os.path

phoonnx/thirdparty/mantoq/__init__.py

@@ -3,14 +3,6 @@ from phoonnx.thirdparty.mantoq.buck.tokenization import (arabic_to_phonemes, pho
                                     phonemes_to_tokens, simplify_phonemes)
 from phoonnx.thirdparty.mantoq.buck.tokenization import tokens_to_ids as _tokens_to_id
 from phoonnx.thirdparty.mantoq.num2words import num2words
-import warnings
-from phoonnx.thirdparty.tashkeel import TashkeelDiacritizer
-try:
-    import onnxruntime
-
-    _TASHKEEL_AVAILABLE = True
-except ImportError:
-    _TASHKEEL_AVAILABLE = False
 
 _DIACRITIZER_INST = None
 
@@ -29,29 +21,12 @@ QUOTES_TABLE = str.maketrans(QUOTES, '"' * len(QUOTES))
 BRACKETS_TABLE = str.maketrans("[]{}", "()()")
 
 
-
-
-def tashkeel(text: str) -> str:
-    global _DIACRITIZER_INST
-    if not _TASHKEEL_AVAILABLE:
-        warnings.warn(
-            "Warning: The Tashkeel feature will not be available. Please re-install with the `libtashkeel` extra.",
-            UserWarning,
-        )
-        return text
-    if _DIACRITIZER_INST is None:
-        _DIACRITIZER_INST = TashkeelDiacritizer()
-    return _DIACRITIZER_INST.diacritize(text)
-
 def g2p(
     text: str,
-    add_tashkeel: bool = True,
     process_numbers: bool = True,
     append_eos: bool = False,
-) -> list[str]:
+) -> tuple[str, list[str]]:
     text = text.translate(AR_SPECIAL_PUNCS_TABLE).translate(QUOTES_TABLE).translate(BRACKETS_TABLE)
-    if add_tashkeel:
-        text = tashkeel(text)
     if process_numbers:
         text = num2words(text)
     normalized_text = text

phoonnx/thirdparty/phonikud/__init__.py

@@ -0,0 +1,24 @@
+import os
+import requests
+
+
+class PhonikudDiacritizer:
+    dl_url = "https://huggingface.co/thewh1teagle/phonikud-onnx/resolve/main/phonikud-1.0.int8.onnx"
+
+    def __init__(self):
+
+        base_path = os.path.expanduser("~/.local/share/phonikud")
+        fname = self.dl_url.split("/")[-1]
+        model = f"{base_path}/{fname}"
+        if not os.path.isfile(model):
+            os.makedirs(base_path, exist_ok=True)
+            # TODO - streaming download
+            data = requests.get(self.dl_url).content
+            with open(model, "wb") as f:
+                f.write(data)
+
+        from phonikud_onnx import Phonikud
+        self.phonikud = Phonikud(model)
+
+    def diacritize(self, text: str) -> str:
+        return self.phonikud.add_diacritics(text)

phoonnx/version.py

@@ -1,6 +1,10 @@
 # START_VERSION_BLOCK
 VERSION_MAJOR = 0
-VERSION_MINOR = 0
-VERSION_BUILD = 2
-VERSION_ALPHA = 2
+VERSION_MINOR = 1
+VERSION_BUILD = 0
+VERSION_ALPHA = 3
 # END_VERSION_BLOCK
+
+VERSION_STR = f"{VERSION_MAJOR}.{VERSION_MINOR}.{VERSION_BUILD}"
+if VERSION_ALPHA:
+    VERSION_STR += f"a{VERSION_ALPHA}"

phoonnx/voice.py

@@ -14,7 +14,6 @@ from phoonnx.config import PhonemeType, VoiceConfig, SynthesisConfig, get_phonem
 from phoonnx.phoneme_ids import phonemes_to_ids, BlankBetween
 from phoonnx.phonemizers import Phonemizer
 from phoonnx.phonemizers.base import PhonemizedChunks
-from phoonnx.thirdparty.tashkeel import TashkeelDiacritizer
 
 _PHONEME_BLOCK_PATTERN = re.compile(r"(\[\[.*?\]\])")
 
@@ -113,11 +112,6 @@ class TTSVoice:
 
     phonemizer: Optional[Phonemizer] = None
 
-    # For Arabic text only
-    use_tashkeel: bool = True
-    tashkeel_diacritizier: Optional[TashkeelDiacritizer] = None  # For Arabic text only
-    taskeen_threshold: Optional[float] = 0.8
-
     def __post_init__(self):
         try:
             self.phonetic_spellings = PhoneticSpellings.from_lang(self.config.lang_code)
@@ -128,10 +122,6 @@ class TTSVoice:
                                              self.config.alphabet,
                                              self.config.phonemizer_model)
 
-        # compat with piper arabic models - TODO move to espeak phonemizer
-        if self.config.lang_code.split("-")[0] == "ar" and self.use_tashkeel and self.tashkeel_diacritizier is None:
-            self.tashkeel_diacritizier = TashkeelDiacritizer()
-
     @staticmethod
     def load(
             model_path: Union[str, Path],
@@ -209,12 +199,6 @@ class TTSVoice:
 
                 continue
 
-            # Arabic diacritization
-            if self.config.lang_code.split("-")[0] == "ar" and self.use_tashkeel:
-                text_part = self.tashkeel_diacritizier(
-                    text_part, taskeen_threshold=self.taskeen_threshold
-                )
-
             # Phonemization
             phonemes = self.phonemizer.phonemize(
                 text_part, self.config.lang_code
@@ -267,6 +251,10 @@ class TTSVoice:
         if self.phonetic_spellings and syn_config.enable_phonetic_spellings:
             text = self.phonetic_spellings.apply(text)
 
+        if syn_config.add_diacritics:
+            text = self.phonemizer.add_diacritics(text, self.config.lang_code)
+            LOG.debug("text+diacritics=%s", text)
+
         # All phonemization goes through the unified self.phonemize method
         sentence_phonemes = self.phonemize(text)
         LOG.debug("phonemes=%s", sentence_phonemes)

{phoonnx-0.0.2a2.dist-info → phoonnx-0.1.0a3.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: phoonnx
-Version: 0.0.2a2
+Version: 0.1.0a3
 Home-page: https://github.com/TigreGotico/phoonnx
 Author: JarbasAi
 Author-email: jarbasai@mailfence.com
@@ -220,6 +220,7 @@ Requires-Dist: librosa<1,>=0.9.2; extra == "train"
 Requires-Dist: numpy<2,>=1.19.0; extra == "train"
 Requires-Dist: pytorch-lightning<2.0; extra == "train"
 Requires-Dist: torch<2,>=1.11.0; extra == "train"
+Requires-Dist: click; extra == "train"
 Provides-Extra: uew
 Requires-Dist: epitran; extra == "uew"
 Provides-Extra: ug

{phoonnx-0.0.2a2.dist-info → phoonnx-0.1.0a3.dist-info}/RECORD

@@ -1,28 +1,29 @@
 phoonnx/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-phoonnx/config.py,sha256=81H34oPG2BaiOA6UM1KapoT341n068LqRprKb5ER6mY,19451
+phoonnx/config.py,sha256=IYhC-kYjLgYmBroId6YeOE2Vp7SMNGtiGqIIe_09NJk,19531
 phoonnx/phoneme_ids.py,sha256=FiNgZwV6naEsBh6XwFLh3_FyOgPiCsK9qo7S0v-CmI4,13667
 phoonnx/util.py,sha256=XSjFEoqSFcujFTHxednacgC9GrSYyF-Il5L6Utmxmu4,25909
-phoonnx/version.py,sha256=E5lPoALEg4bqb4bmnVYnseCgozkzjZGnhpUom_OlcQI,114
-phoonnx/voice.py,sha256=FR_LafK1vSi_anPERJjZBuH3Bb9vUIof0MAW6TnALlA,20024
+phoonnx/version.py,sha256=WnY5J2wtSTore9QbKwfk04gQhBsYq4HVmV5CBjEhGnk,236
+phoonnx/voice.py,sha256=JXjmbrhJd4mmTiLgz4O_Pa5_rKGUC9xzuBfqxYDw3Mg,19420
 phoonnx/locale/ca/phonetic_spellings.txt,sha256=igv3t7jxLSRE5GHsdn57HOpxiWNcEmECPql6m02wbO0,47
 phoonnx/locale/en/phonetic_spellings.txt,sha256=xGQlWOABLzbttpQvopl9CU-NnwEJRqKx8iuylsdUoQA,27
 phoonnx/locale/gl/phonetic_spellings.txt,sha256=igv3t7jxLSRE5GHsdn57HOpxiWNcEmECPql6m02wbO0,47
 phoonnx/locale/pt/phonetic_spellings.txt,sha256=KntS8QMynEJ5A3Clvcjq4qlmL-ThSbhfD6v0nKSrlqs,49
 phoonnx/phonemizers/__init__.py,sha256=QGBZk0QUgJdg2MwUWY9Kpk6ucwrEJYtHb07YcNvXCV4,1647
-phoonnx/phonemizers/ar.py,sha256=rPAMGPlyXOlKY99IoHe_vWcLllepiG0YFUbOBia075E,1260
-phoonnx/phonemizers/base.py,sha256=yPg6-dvscYpl3rR3JEULG1PRF-i8DWC_C3HAZGLbxOo,7648
+phoonnx/phonemizers/ar.py,sha256=xxILq5iyH0kcI-NqFfRK4abGtpdUbykBjt_dZmPuO2w,3216
+phoonnx/phonemizers/base.py,sha256=FHvAsvSjAl_oSa1GoeEi96CQ_JO_xkKXWq0ukuMxiuo,8660
 phoonnx/phonemizers/en.py,sha256=N2SVoVhplQao7Ej5TXbxJU-YkAgkY0Fr9iYBFnsjFSE,9271
 phoonnx/phonemizers/fa.py,sha256=d_DZM2wqomf4gcRH_rFcNA3VkQWKHru8vwBwaNG8Ll8,1452
 phoonnx/phonemizers/gl.py,sha256=jEFKJJViHufZtB7lGNwWQCdWGiNKDCVZ_GRYXTaw_2c,6614
-phoonnx/phonemizers/he.py,sha256=KbRI3XRZa8UtJdNWmn_fd-t5lmFSIp4Mw8UgcO5l-Po,2211
+phoonnx/phonemizers/he.py,sha256=49OFS34wSFvvR9B3z2bGSzSLmlIvnn2HtkHBOkHS9Ns,1383
 phoonnx/phonemizers/ja.py,sha256=Xojsrt715ihnIiEk9K6giYqDo9Iykw-SHfIidrHtHSU,3834
 phoonnx/phonemizers/ko.py,sha256=kwWoOFqanCB8kv2JRx17A0hP78P1wbXlX6e8VBn1ezQ,2989
-phoonnx/phonemizers/mul.py,sha256=37G_G58aGnVpdEm9vZEAOdGEHJ9TLBE17bU1HFvQ2rU,27291
+phoonnx/phonemizers/mul.py,sha256=-h6uN_laUD-unNRGThzjyiOZpN6pSl4uinCndg5-0TA,94184
 phoonnx/phonemizers/vi.py,sha256=_XJc-Xeawr1Lxr7o8mE_hJao1aGcj4g01XYAOxC_Scg,1311
 phoonnx/phonemizers/zh.py,sha256=88Ywq8h9LDanlyz8RHjRSCY_PRK_Dq808tBADyrgaP8,9657
 phoonnx/thirdparty/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 phoonnx/thirdparty/arpa2ipa.py,sha256=Uj1G5NgP5oBBfSm26LGB8QoumdT-NqCLQTZHT165-_o,5850
-phoonnx/thirdparty/hangul2ipa.py,sha256=e2c0WOy5lFMcf6GS7pNqIbauMKBX07S84lCczZAZJGA,27518
+phoonnx/thirdparty/bw2ipa.py,sha256=5FiWC4AP4KXkqtbclbinoXEsUnSYEjk4VWAPasMMcbg,2328
+phoonnx/thirdparty/hangul2ipa.py,sha256=Pj06lL-GkOH4ZkLuakwQAT045fEVsijGhwoY_EEEVKc,27572
 phoonnx/thirdparty/zh_num.py,sha256=SESA6gvSJW3LZ0FLoybXn2SpbxqhQTi9Tg_U2IZ5JYY,7147
 phoonnx/thirdparty/cotovia/cotovia_aarch64,sha256=BsAWZN452Lm9kDU4i6rQGHFSlmxP3GfHRKhbJMUQrfA,6764592
 phoonnx/thirdparty/cotovia/cotovia_x86_64,sha256=-6BNx_cd49nnDreOAsGtVtePs_X76esrqcNAfmksN1o,1379832
@@ -36,7 +37,7 @@ phoonnx/thirdparty/ko_tables/tensification.csv,sha256=V4Xf3A1G1iMBzwZevBKQuk_lPa
 phoonnx/thirdparty/ko_tables/yale.csv,sha256=UhtDbPXRAAyAKoQMXmwhVBwJ5pfZQ_Duk28qBtRUdsU,297
 phoonnx/thirdparty/kog2p/__init__.py,sha256=yLizadg7RXM-3dQyftD4XSk8r2jb0QOlHQ6as9uUa4U,10267
 phoonnx/thirdparty/kog2p/rulebook.txt,sha256=FQE3nej8wojl6ilVUBYo7f8bIk0Hjci-B7HPXhM-xNc,9303
-phoonnx/thirdparty/mantoq/__init__.py,sha256=4kZuZ3RA5ZhQwTOQGkHF9jQYSvetNTn9uWi5Dsx101k,2106
+phoonnx/thirdparty/mantoq/__init__.py,sha256=02FftO4Onmp_S-XdukbBQ3aRVvqEQyo1frCLWgcF9cY,1428
 phoonnx/thirdparty/mantoq/num2words.py,sha256=9-ncMtxV1FusD9rNur1lu7l2DWhwUwI1mFiqiPSMH_Q,1264
 phoonnx/thirdparty/mantoq/unicode_symbol2label.py,sha256=CeZNv7qWeQS4Ejvz-sKgK--5eNYdVVv04WHPaOeK4gk,259409
 phoonnx/thirdparty/mantoq/buck/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
@@ -52,6 +53,7 @@ phoonnx/thirdparty/mantoq/pyarabic/number.py,sha256=NjFZPWRu-9dZDLgxfv9oDjmh-kWY
 phoonnx/thirdparty/mantoq/pyarabic/number_const.py,sha256=vAvRVENxTrl9gWPllSXF-yqK9fAW6htuA2d041btC_A,42361
 phoonnx/thirdparty/mantoq/pyarabic/stack.py,sha256=aJeSzQxVNdomDTWXuxIXWXVOc2BW_3iRWnwmBLkB8jM,1022
 phoonnx/thirdparty/mantoq/pyarabic/trans.py,sha256=cusyHk9Y01iuvMLJXxgCnIiGyAORzEdSosDKX4cAhPc,13713
+phoonnx/thirdparty/phonikud/__init__.py,sha256=g1dCelCZbwlKT0Ibaky6Ckp59wMw5g_1DDyDXauqFTg,760
 phoonnx/thirdparty/tashkeel/LICENSE,sha256=mQjTJ6MGAXzmYkO7x4O2VuEeSwCMx7lncbc26TnrVjw,1067
 phoonnx/thirdparty/tashkeel/SOURCE,sha256=SmnRz-Am5EXv-n2-RokJVEhnn8zeF1QZJVvMQDA_Qds,38
 phoonnx/thirdparty/tashkeel/__init__.py,sha256=FRdGNCTQaai9X077vlNh4tFOvWgm1U2lIUgnQKO5q0s,7119
@@ -60,8 +62,8 @@ phoonnx/thirdparty/tashkeel/input_id_map.json,sha256=cnpJqjx-k53AbzKyfC4GxMS771l
 phoonnx/thirdparty/tashkeel/model.onnx,sha256=UsQNQsoJT_n_B6CR0KHq_XuqXPI4jmCpzIm6zY5elV8,4788213
 phoonnx/thirdparty/tashkeel/target_id_map.json,sha256=baNAJL_UwP9U91mLt01aAEBRRNdGr-csFB_O6roh7TA,181
 phoonnx_train/__main__.py,sha256=FUAIsbQ-w2i_hoNiBuriQFk4uoryhL4ydyVY-hVjw1U,5086
-phoonnx_train/export_onnx.py,sha256=dcFJRZl4YvBk_Dj3j0aNAQVEqKfBHTzV22pzvQwSETQ,2909
-phoonnx_train/preprocess.py,sha256=0kto9Holywby6lnoQucBXq2wYEKDItRvdkvYbQnLJeo,14447
+phoonnx_train/export_onnx.py,sha256=CPfgNEm0hnXPSlgme0R9jr-6jZ5fKFpG5DZJFMkC-h4,12820
+phoonnx_train/preprocess.py,sha256=8_Opy5QVNjVmSVmh1_IF23bcNebVIEXuK2KcollIy28,15793
 phoonnx_train/norm_audio/__init__.py,sha256=Al_YwqMnENXRWp0c79cDZqbdd7pFYARXKxCfBaedr1c,3030
 phoonnx_train/norm_audio/trim.py,sha256=_ZsE3SYhahQSdEdBLeSwyFJGcvEbt-5E_lnWwTT4tcY,1698
 phoonnx_train/norm_audio/vad.py,sha256=DXHfRD0qqFJ52FjPvrL5LlN6keJWuc9Nf6TNhxpwC_4,1600
@@ -69,7 +71,7 @@ phoonnx_train/vits/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuF
 phoonnx_train/vits/attentions.py,sha256=yc_ViF8zR8z68DzphmVVVn27f9xK_5wi8S4ITLXVQL0,15134
 phoonnx_train/vits/commons.py,sha256=JsD8CdZ3ZcYYubYhw8So5hICBziFlCrKLrv1lMDRCDM,4645
 phoonnx_train/vits/config.py,sha256=oSuUIhw9Am7BQ5JwDgtCO-P1zRyN7nPgR-U1XuncJls,10789
-phoonnx_train/vits/dataset.py,sha256=DLLGSCkn3GF9uktoTprH1ERblZ18GO6-QsClQKWa98o,6804
+phoonnx_train/vits/dataset.py,sha256=1V1tVh5dSLjFMBsuzrAsoGtYWSBT4iU64Jdqi8oG-y0,7016
 phoonnx_train/vits/lightning.py,sha256=ZBuSIiJ7EUU1Za2V8Uh6-_HGGRW_qwpXLLs1cEDirHA,12301
 phoonnx_train/vits/losses.py,sha256=j-uINhBcYxVXFvFutiewQpTuw-qF-J6M6hdJVeOKqNE,1401
 phoonnx_train/vits/mel_processing.py,sha256=huIjbQgewSmM39hdzRZvZUCI7fTNSMmLcAv3f8zYb8k,3956
@@ -80,7 +82,7 @@ phoonnx_train/vits/utils.py,sha256=exiyrtPHbnnGvcHWSbaH9-gR6srH5ZPHlKiqV2IHUrQ,4
 phoonnx_train/vits/wavfile.py,sha256=oQZiTIrdw0oLTbcVwKfGXye1WtKte6qK_52qVwiMvfc,26396
 phoonnx_train/vits/monotonic_align/__init__.py,sha256=5IdAOD1Z7UloMb6d_9NRFsXoNIjEQ3h9mvOSh_AtO3k,636
 phoonnx_train/vits/monotonic_align/setup.py,sha256=0K5iJJ2mKIklx6ncEfCQS34skm5hHPiz9vRlQEvevvY,266
-phoonnx-0.0.2a2.dist-info/METADATA,sha256=-uMrs5iV1A4gqenGg1r8ZyhZN1DhBd9HCsezIy4UxPE,8145
-phoonnx-0.0.2a2.dist-info/WHEEL,sha256=tZoeGjtWxWRfdplE7E3d45VPlLNQnvbKiYnx7gwAy8A,92
-phoonnx-0.0.2a2.dist-info/top_level.txt,sha256=ZrnHXe-4HqbOSX6fbdY-JiP7YEu2Bok9T0ji351MrmM,22
-phoonnx-0.0.2a2.dist-info/RECORD,,
+phoonnx-0.1.0a3.dist-info/METADATA,sha256=3U1Ea0g2HxtWPsIs7NCxzPdo7ZTr4s_lRs9gIOC6MWY,8184
+phoonnx-0.1.0a3.dist-info/WHEEL,sha256=tZoeGjtWxWRfdplE7E3d45VPlLNQnvbKiYnx7gwAy8A,92
+phoonnx-0.1.0a3.dist-info/top_level.txt,sha256=ZrnHXe-4HqbOSX6fbdY-JiP7YEu2Bok9T0ji351MrmM,22
+phoonnx-0.1.0a3.dist-info/RECORD,,