phoonnx 0.0.2a2__py3-none-any.whl → 0.1.0a3__py3-none-any.whl

phoonnx/config.py

@@ -33,7 +33,7 @@ class Alphabet(str, Enum):
     ERAAB = "eraab" # fa
     COTOVIA = "cotovia" # gl
     HANZI = "hanzi" # zh
-    MANTOQ = "mantoq" # ar
+    BUCKWALTER = "buckwalter" # ar
 
 
 
@@ -382,6 +382,9 @@ class SynthesisConfig:
 
     enable_phonetic_spellings: bool = True
 
+    """for arabic and hebrew models"""
+    add_diacritics: bool = True
+
 
 def get_phonemizer(phoneme_type: PhonemeType,
                    alphabet: Alphabet = Alphabet.IPA,

phoonnx/phonemizers/ar.py

@@ -1,11 +1,15 @@
+from phoonnx.config import Alphabet
 from phoonnx.phonemizers.base import BasePhonemizer
+from phoonnx.thirdparty.bw2ipa import translate as bw2ipa
 from phoonnx.thirdparty.mantoq import g2p as mantoq
-from phoonnx.config import Alphabet
+
 
 class MantoqPhonemizer(BasePhonemizer):
 
-    def __init__(self):
-        super().__init__(Alphabet.MANTOQ)
+    def __init__(self, alphabet=Alphabet.BUCKWALTER):
+        if alphabet not in [Alphabet.IPA, Alphabet.BUCKWALTER]:
+            raise ValueError("unsupported alphabet")
+        super().__init__(alphabet)
 
     @classmethod
     def get_lang(cls, target_lang: str) -> str:
@@ -26,17 +30,69 @@ class MantoqPhonemizer(BasePhonemizer):
 
     def phonemize_string(self, text: str, lang: str = "ar") -> str:
         """
+        Phonemizes an Arabic string using the Mantoq G2P tool.
+        If the alphabet is set to IPA, it then converts the result using bw2ipa.
         """
         lang = self.get_lang(lang)
+        # The mantoq function returns a tuple of (normalized_text, phonemes)
         normalized_text, phonemes = mantoq(text)
-        return "".join(phonemes).replace("_+_", " ")
+
+        # The phonemes are a list of characters, we join them into a string
+        # and replace the word separator token with a space.
+        phonemes = "".join(phonemes).replace("_+_", " ")
+
+        if self.alphabet == Alphabet.IPA:
+            # If the alphabet is IPA, we use the bw2ipa function to translate
+            # the Buckwalter-like phonemes into IPA.
+            return bw2ipa(phonemes)
+
+        # Otherwise, we return the phonemes in the default Mantoq alphabet.
+        return phonemes
 
 
 if __name__ == "__main__":
+    from phoonnx.phonemizers.mul import EspeakPhonemizer
+
+    espeak = EspeakPhonemizer()
+
+    # Initialize phonemizers for both MANTOQ and IPA alphabets
+    pho_mantoq = MantoqPhonemizer(alphabet=Alphabet.IPA)
+
+
+    def compare(text):
+        print(f"Original Text: {text}")
+        print(f"  Mantoq: {pho_mantoq.phonemize_string(text, 'ar')}")
+        print(f"  Espeak: {espeak.phonemize_string(text, 'ar')}")
+
+        ts = pho_mantoq.add_diacritics(text, 'ar')
+        print(f"Tashkeel Text: {ts}")
+        print(f"  Mantoq: {pho_mantoq.phonemize_string(ts, 'ar')}")
+        print(f"  Espeak: {espeak.phonemize_string(ts, 'ar')}")
+        print("\n#########################")
+
+
     text = "مرحبا بالعالم"
-    # gets normalized to
-    # مَرْحَبًا بِالْعالَم
+    compare(text)
+
+    text = "ذهب الطالب إلى المكتبة لقراءة كتاب عن تاريخ الأندلس."
+    compare(text)
+
+    # 1. Test for gemination of a sun letter (e.g., ash-shams)
+    text = "الشمس"
+    compare(text)
+
+    # 2. Test for long vowels (e.g., 'fil' - elephant)
+    text = "فيل"
+    compare(text)
+
+    # 3. Test for glide (e.g., 'yawm' - day)
+    text = "يوم"
+    compare(text)
+
+    # 4. Test for long vowels (e.g., 'suwr' - wall)
+    text = "سور"
+    compare(text)
 
-    pho = MantoqPhonemizer()
-    print(pho.phonemize(text, "ar"))
-    # [('m a r H a b a n aa   b i l E aa l a m', '.', True)]
+    # 5. Test for glide (e.g., 'law' - if)
+    text = "لو"
+    compare(text)

phoonnx/phonemizers/base.py

@@ -8,6 +8,8 @@ from langcodes import tag_distance
 from quebra_frases import sentence_tokenize
 from phoonnx.config import Alphabet
 from phoonnx.util import normalize
+from phoonnx.thirdparty.phonikud import PhonikudDiacritizer
+from phoonnx.thirdparty.tashkeel import TashkeelDiacritizer
 
 # list of (substring, terminator, end_of_sentence) tuples.
 TextChunks = List[Tuple[str, str, bool]]
@@ -18,10 +20,27 @@ PhonemizedChunks = list[list[str]]
 
 
 class BasePhonemizer(metaclass=abc.ABCMeta):
-    def __init__(self, alphabet: Alphabet = Alphabet.UNICODE):
+    def __init__(self, alphabet: Alphabet = Alphabet.UNICODE,
+                 taskeen_threshold: Optional[float] = 0.8):
         super().__init__()
         self.alphabet = alphabet
 
+        self.taskeen_threshold = taskeen_threshold  # arabic only
+        self._tashkeel: Optional[TashkeelDiacritizer] = None
+        self._phonikud: Optional[PhonikudDiacritizer] = None # hebrew only
+
+    @property
+    def phonikud(self) -> PhonikudDiacritizer:
+        if self._phonikud is None:
+            self._phonikud = PhonikudDiacritizer()
+        return self._phonikud
+
+    @property
+    def tashkeel(self) -> TashkeelDiacritizer:
+        if self._tashkeel is None:
+            self._tashkeel = TashkeelDiacritizer()
+        return self._tashkeel
+
     @abc.abstractmethod
     def phonemize_string(self, text: str, lang: str) -> str:
         raise NotImplementedError
@@ -29,6 +48,13 @@ class BasePhonemizer(metaclass=abc.ABCMeta):
     def phonemize_to_list(self, text: str, lang: str) -> List[str]:
         return list(self.phonemize_string(text, lang))
 
+    def add_diacritics(self, text: str, lang: str) -> str:
+        if lang.startswith("he"):
+            return self.phonikud.diacritize(text)
+        elif lang.startswith("ar"):
+            return self.tashkeel.diacritize(text, self.taskeen_threshold)
+        return text
+
     def phonemize(self, text: str, lang: str) -> PhonemizedChunks:
         if not text:
             return [('', '', True)]

phoonnx/phonemizers/he.py

@@ -1,30 +1,12 @@
-import os.path
-
-import requests
-
-from phoonnx.phonemizers.base import BasePhonemizer
 from phoonnx.config import Alphabet
+from phoonnx.phonemizers.base import BasePhonemizer
 
 
 class PhonikudPhonemizer(BasePhonemizer):
-    dl_url = "https://huggingface.co/thewh1teagle/phonikud-onnx/resolve/main/phonikud-1.0.int8.onnx"
 
-    def __init__(self, model: str = None, diacritics=True):
-        from phonikud_onnx import Phonikud
+    def __init__(self):
         from phonikud import phonemize
         self.g2p = phonemize
-        self.diacritics = diacritics
-        if model is None:
-            base_path = os.path.expanduser("~/.local/share/phonikud")
-            fname = self.dl_url.split("/")[-1]
-            model = f"{base_path}/{fname}"
-            if not os.path.isfile(model):
-                os.makedirs(base_path, exist_ok=True)
-                # TODO - streaming download
-                data = requests.get(self.dl_url).content
-                with open(model, "wb") as f:
-                    f.write(data)
-        self.phonikud = Phonikud(model) if diacritics else None
         super().__init__(Alphabet.IPA)
 
     @classmethod
@@ -48,20 +30,19 @@ class PhonikudPhonemizer(BasePhonemizer):
         """
         """
         lang = self.get_lang(lang)
-        if self.diacritics:
-            text = self.phonikud.add_diacritics(text)
         return self.g2p(text)
 
 
 if __name__ == "__main__":
-    #text = "מתכת יקרה"
+    # text = "מתכת יקרה"
     text = 'שָׁלוֹם עוֹלָם'
 
-    pho = PhonikudPhonemizer(diacritics=False)
+    pho = PhonikudPhonemizer()
     lang = "he"
 
     print(f"\n--- Getting phonemes for '{text}' ---")
+    # text = pho.add_diacritics(text, lang)
     phonemes = pho.phonemize(text, lang)
     print(f"  Phonemes: {phonemes}")
     # --- Getting phonemes for 'שָׁלוֹם עוֹלָם' ---
-    #   Phonemes: [('ʃalˈom ʔolˈam', '.', True)]
+    #   Phonemes: [('ʃalˈom ʔolˈam', '.', True)]