phoonnx 0.1.0a1__py3-none-any.whl → 0.1.1a1__py3-none-any.whl

phoonnx/config.py

@@ -33,7 +33,7 @@ class Alphabet(str, Enum):
     ERAAB = "eraab" # fa
     COTOVIA = "cotovia" # gl
     HANZI = "hanzi" # zh
-    MANTOQ = "mantoq" # ar
+    BUCKWALTER = "buckwalter" # ar
 
 
 
@@ -48,6 +48,8 @@ class PhonemeType(str, Enum):
     EPITRAN = "epitran"
     BYT5 = "byt5"
     CHARSIU = "charsiu"  # technically same as byt5, but needs special handling for whitespace
+    TRANSPHONE = "transphone"
+    MIRANDESE = "mwl_phonemizer"
 
     DEEPPHONEMIZER = "deepphonemizer" # en
     OPENPHONEMIZER = "openphonemizer" # en
@@ -382,6 +384,9 @@ class SynthesisConfig:
 
     enable_phonetic_spellings: bool = True
 
+    """for arabic and hebrew models"""
+    add_diacritics: bool = True
+
 
 def get_phonemizer(phoneme_type: PhonemeType,
                    alphabet: Alphabet = Alphabet.IPA,
@@ -389,6 +394,7 @@ def get_phonemizer(phoneme_type: PhonemeType,
     from phoonnx.phonemizers import (EpitranPhonemizer, EspeakPhonemizer, OpenPhonemizer, OpenJTaklPhonemizer,
                                      ByT5Phonemizer, CharsiuPhonemizer, DeepPhonemizer, PersianPhonemizer,
                                      G2pCPhonemizer, G2pMPhonemizer, G2PKPhonemizer, G2PEnPhonemizer,
+                                     TransphonePhonemizer, MirandesePhonemizer,
                                      GruutPhonemizer, GraphemePhonemizer, MantoqPhonemizer, MisakiPhonemizer,
                                      KoG2PPhonemizer, PypinyinPhonemizer, PyKakasiPhonemizer, CotoviaPhonemizer,
                                      CutletPhonemizer, PhonikudPhonemizer, VIPhonemePhonemizer, XpinyinPhonemizer,
@@ -405,6 +411,10 @@ def get_phonemizer(phoneme_type: PhonemeType,
         phonemizer = EpitranPhonemizer()
     elif phoneme_type == PhonemeType.MISAKI:
         phonemizer = MisakiPhonemizer()
+    elif phoneme_type == PhonemeType.TRANSPHONE:
+        phonemizer = TransphonePhonemizer()
+    elif phoneme_type == PhonemeType.MIRANDESE:
+        phonemizer = MirandesePhonemizer()
     elif phoneme_type == PhonemeType.DEEPPHONEMIZER:
         phonemizer = DeepPhonemizer(model)
     elif phoneme_type == PhonemeType.OPENPHONEMIZER:

phoonnx/phonemizers/__init__.py

@@ -12,7 +12,8 @@ from phoonnx.phonemizers.ko import KoG2PPhonemizer, G2PKPhonemizer
 from phoonnx.phonemizers.zh import (G2pCPhonemizer, G2pMPhonemizer, PypinyinPhonemizer,
                                     XpinyinPhonemizer, JiebaPhonemizer)
 from phoonnx.phonemizers.mul import (EspeakPhonemizer, EpitranPhonemizer, MisakiPhonemizer,
-                                     GruutPhonemizer, ByT5Phonemizer, CharsiuPhonemizer)
+                                     GruutPhonemizer, ByT5Phonemizer, CharsiuPhonemizer, TransphonePhonemizer)
+from phoonnx.phonemizers.mwl import MirandesePhonemizer
 
 Phonemizer = Union[
     MisakiPhonemizer,
@@ -22,6 +23,8 @@ Phonemizer = Union[
     EspeakPhonemizer,
     GruutPhonemizer,
     EpitranPhonemizer,
+    TransphonePhonemizer,
+    MirandesePhonemizer,
     OpenJTaklPhonemizer,
     CutletPhonemizer,
     PyKakasiPhonemizer,

phoonnx/phonemizers/ar.py

@@ -1,13 +1,13 @@
-from phoonnx.phonemizers.base import BasePhonemizer
-from phoonnx.thirdparty.mantoq import g2p as mantoq
 from phoonnx.config import Alphabet
+from phoonnx.phonemizers.base import BasePhonemizer
 from phoonnx.thirdparty.bw2ipa import translate as bw2ipa
+from phoonnx.thirdparty.mantoq import g2p as mantoq
 
 
 class MantoqPhonemizer(BasePhonemizer):
 
-    def __init__(self, alphabet=Alphabet.MANTOQ):
-        if alphabet not in [Alphabet.IPA, Alphabet.MANTOQ]:
+    def __init__(self, alphabet=Alphabet.BUCKWALTER):
+        if alphabet not in [Alphabet.IPA, Alphabet.BUCKWALTER]:
             raise ValueError("unsupported alphabet")
         super().__init__(alphabet)
 
@@ -39,7 +39,7 @@ class MantoqPhonemizer(BasePhonemizer):
 
         # The phonemes are a list of characters, we join them into a string
         # and replace the word separator token with a space.
-        phonemes =  "".join(phonemes).replace("_+_", " ")
+        phonemes = "".join(phonemes).replace("_+_", " ")
 
         if self.alphabet == Alphabet.IPA:
             # If the alphabet is IPA, we use the bw2ipa function to translate
@@ -51,56 +51,48 @@ class MantoqPhonemizer(BasePhonemizer):
 
 
 if __name__ == "__main__":
+    from phoonnx.phonemizers.mul import EspeakPhonemizer
+
+    espeak = EspeakPhonemizer()
+
     # Initialize phonemizers for both MANTOQ and IPA alphabets
-    pho_mantoq = MantoqPhonemizer()
-    pho_ipa = MantoqPhonemizer(alphabet=Alphabet.IPA)
+    pho_mantoq = MantoqPhonemizer(alphabet=Alphabet.IPA)
+
+
+    def compare(text):
+        print(f"Original Text: {text}")
+        print(f"  Mantoq: {pho_mantoq.phonemize_string(text, 'ar')}")
+        print(f"  Espeak: {espeak.phonemize_string(text, 'ar')}")
+
+        ts = pho_mantoq.add_diacritics(text, 'ar')
+        print(f"Tashkeel Text: {ts}")
+        print(f"  Mantoq: {pho_mantoq.phonemize_string(ts, 'ar')}")
+        print(f"  Espeak: {espeak.phonemize_string(ts, 'ar')}")
+        print("\n#########################")
 
-    text1 = "مرحبا بالعالم"
-    print(f"Original Text: {text1}")
-    print(f"  Mantoq Phonemizer: {pho_mantoq.phonemize_string(text1, 'ar')}")
-    print(f"  IPA Phonemizer:    {pho_ipa.phonemize_string(text1, 'ar')}")
-    print("-" * 20)
 
-    text2 = "ذهب الطالب إلى المكتبة لقراءة كتاب عن تاريخ الأندلس."
-    print(f"Original Text: {text2}")
-    print(f"  Mantoq Phonemizer: {pho_mantoq.phonemize_string(text2, 'ar')}")
-    print(f"  IPA Phonemizer:    {pho_ipa.phonemize_string(text2, 'ar')}")
-    print("-" * 20)
+    text = "مرحبا بالعالم"
+    compare(text)
 
-    # --- New Test Cases to check bw2ipa logic ---
-    print("--- New Test Cases for bw2ipa logic ---")
+    text = "ذهب الطالب إلى المكتبة لقراءة كتاب عن تاريخ الأندلس."
+    compare(text)
 
     # 1. Test for gemination of a sun letter (e.g., ash-shams)
-    text3 = "الشمس"
-    print(f"Original Text: '{text3}'")
-    print(f"  Mantoq Phonemizer: {pho_mantoq.phonemize_string(text3, 'ar')}")
-    print(f"  IPA Phonemizer:    {pho_ipa.phonemize_string(text3, 'ar')}")
-    print("-" * 20)
+    text = "الشمس"
+    compare(text)
 
     # 2. Test for long vowels (e.g., 'fil' - elephant)
-    text4 = "فيل"
-    print(f"Original Text: '{text4}'")
-    print(f"  Mantoq Phonemizer: {pho_mantoq.phonemize_string(text4, 'ar')}")
-    print(f"  IPA Phonemizer:    {pho_ipa.phonemize_string(text4, 'ar')}")
-    print("-" * 20)
+    text = "فيل"
+    compare(text)
 
     # 3. Test for glide (e.g., 'yawm' - day)
-    text5 = "يوم"
-    print(f"Original Text: '{text5}'")
-    print(f"  Mantoq Phonemizer: {pho_mantoq.phonemize_string(text5, 'ar')}")
-    print(f"  IPA Phonemizer:    {pho_ipa.phonemize_string(text5, 'ar')}")
-    print("-" * 20)
+    text = "يوم"
+    compare(text)
 
     # 4. Test for long vowels (e.g., 'suwr' - wall)
-    text6 = "سور"
-    print(f"Original Text: '{text6}'")
-    print(f"  Mantoq Phonemizer: {pho_mantoq.phonemize_string(text6, 'ar')}")
-    print(f"  IPA Phonemizer:    {pho_ipa.phonemize_string(text6, 'ar')}")
-    print("-" * 20)
+    text = "سور"
+    compare(text)
 
     # 5. Test for glide (e.g., 'law' - if)
-    text7 = "لو"
-    print(f"Original Text: '{text7}'")
-    print(f"  Mantoq Phonemizer: {pho_mantoq.phonemize_string(text7, 'ar')}")
-    print(f"  IPA Phonemizer:    {pho_ipa.phonemize_string(text7, 'ar')}")
-    print("-" * 20)
+    text = "لو"
+    compare(text)

phoonnx/phonemizers/base.py

@@ -8,6 +8,8 @@ from langcodes import tag_distance
 from quebra_frases import sentence_tokenize
 from phoonnx.config import Alphabet
 from phoonnx.util import normalize
+from phoonnx.thirdparty.phonikud import PhonikudDiacritizer
+from phoonnx.thirdparty.tashkeel import TashkeelDiacritizer
 
 # list of (substring, terminator, end_of_sentence) tuples.
 TextChunks = List[Tuple[str, str, bool]]
@@ -18,10 +20,27 @@ PhonemizedChunks = list[list[str]]
 
 
 class BasePhonemizer(metaclass=abc.ABCMeta):
-    def __init__(self, alphabet: Alphabet = Alphabet.UNICODE):
+    def __init__(self, alphabet: Alphabet = Alphabet.UNICODE,
+                 taskeen_threshold: Optional[float] = 0.8):
         super().__init__()
         self.alphabet = alphabet
 
+        self.taskeen_threshold = taskeen_threshold  # arabic only
+        self._tashkeel: Optional[TashkeelDiacritizer] = None
+        self._phonikud: Optional[PhonikudDiacritizer] = None # hebrew only
+
+    @property
+    def phonikud(self) -> PhonikudDiacritizer:
+        if self._phonikud is None:
+            self._phonikud = PhonikudDiacritizer()
+        return self._phonikud
+
+    @property
+    def tashkeel(self) -> TashkeelDiacritizer:
+        if self._tashkeel is None:
+            self._tashkeel = TashkeelDiacritizer()
+        return self._tashkeel
+
     @abc.abstractmethod
     def phonemize_string(self, text: str, lang: str) -> str:
         raise NotImplementedError
@@ -29,6 +48,13 @@ class BasePhonemizer(metaclass=abc.ABCMeta):
     def phonemize_to_list(self, text: str, lang: str) -> List[str]:
         return list(self.phonemize_string(text, lang))
 
+    def add_diacritics(self, text: str, lang: str) -> str:
+        if lang.startswith("he"):
+            return self.phonikud.diacritize(text)
+        elif lang.startswith("ar"):
+            return self.tashkeel.diacritize(text, self.taskeen_threshold)
+        return text
+
     def phonemize(self, text: str, lang: str) -> PhonemizedChunks:
         if not text:
             return [('', '', True)]

phoonnx/phonemizers/he.py

@@ -1,30 +1,12 @@
-import os.path
-
-import requests
-
-from phoonnx.phonemizers.base import BasePhonemizer
 from phoonnx.config import Alphabet
+from phoonnx.phonemizers.base import BasePhonemizer
 
 
 class PhonikudPhonemizer(BasePhonemizer):
-    dl_url = "https://huggingface.co/thewh1teagle/phonikud-onnx/resolve/main/phonikud-1.0.int8.onnx"
 
-    def __init__(self, model: str = None, diacritics=True):
-        from phonikud_onnx import Phonikud
+    def __init__(self):
         from phonikud import phonemize
         self.g2p = phonemize
-        self.diacritics = diacritics
-        if model is None:
-            base_path = os.path.expanduser("~/.local/share/phonikud")
-            fname = self.dl_url.split("/")[-1]
-            model = f"{base_path}/{fname}"
-            if not os.path.isfile(model):
-                os.makedirs(base_path, exist_ok=True)
-                # TODO - streaming download
-                data = requests.get(self.dl_url).content
-                with open(model, "wb") as f:
-                    f.write(data)
-        self.phonikud = Phonikud(model) if diacritics else None
         super().__init__(Alphabet.IPA)
 
     @classmethod
@@ -48,20 +30,19 @@ class PhonikudPhonemizer(BasePhonemizer):
         """
         """
         lang = self.get_lang(lang)
-        if self.diacritics:
-            text = self.phonikud.add_diacritics(text)
         return self.g2p(text)
 
 
 if __name__ == "__main__":
-    #text = "מתכת יקרה"
+    # text = "מתכת יקרה"
     text = 'שָׁלוֹם עוֹלָם'
 
-    pho = PhonikudPhonemizer(diacritics=False)
+    pho = PhonikudPhonemizer()
     lang = "he"
 
     print(f"\n--- Getting phonemes for '{text}' ---")
+    # text = pho.add_diacritics(text, lang)
     phonemes = pho.phonemize(text, lang)
     print(f"  Phonemes: {phonemes}")
     # --- Getting phonemes for 'שָׁלוֹם עוֹלָם' ---
-    #   Phonemes: [('ʃalˈom ʔolˈam', '.', True)]
+    #   Phonemes: [('ʃalˈom ʔolˈam', '.', True)]