phoonnx 0.0.2a1__py3-none-any.whl → 0.1.0__py3-none-any.whl

phoonnx/config.py

@@ -6,7 +6,6 @@ from phoonnx.phoneme_ids import (load_phoneme_ids, BlankBetween,
                                  DEFAULT_BLANK_WORD_TOKEN, DEFAULT_BLANK_TOKEN,
                                  DEFAULT_PAD_TOKEN, DEFAULT_BOS_TOKEN, DEFAULT_EOS_TOKEN)
 
-
 DEFAULT_NOISE_SCALE = 0.667
 DEFAULT_LENGTH_SCALE = 1.0
 DEFAULT_NOISE_W_SCALE = 0.8
@@ -22,6 +21,8 @@ class Alphabet(str, Enum):
     UNICODE = "unicode"
     IPA = "ipa"
     ARPA = "arpa" # en
+    SAMPA = "sampa"
+    XSAMPA = "x-sampa"
     HANGUL = "hangul" # ko
     KANA = "kana" # ja
     HIRA = "hira" # ja
@@ -32,6 +33,7 @@ class Alphabet(str, Enum):
     ERAAB = "eraab" # fa
     COTOVIA = "cotovia" # gl
     HANZI = "hanzi" # zh
+    BUCKWALTER = "buckwalter" # ar
 
 
 
@@ -380,6 +382,9 @@ class SynthesisConfig:
 
     enable_phonetic_spellings: bool = True
 
+    """for arabic and hebrew models"""
+    add_diacritics: bool = True
+
 
 def get_phonemizer(phoneme_type: PhonemeType,
                    alphabet: Alphabet = Alphabet.IPA,

phoonnx/phonemizers/ar.py

@@ -1,11 +1,15 @@
+from phoonnx.config import Alphabet
 from phoonnx.phonemizers.base import BasePhonemizer
+from phoonnx.thirdparty.bw2ipa import translate as bw2ipa
 from phoonnx.thirdparty.mantoq import g2p as mantoq
-from phoonnx.config import Alphabet
+
 
 class MantoqPhonemizer(BasePhonemizer):
 
-    def __init__(self):
-        super().__init__(Alphabet.IPA)
+    def __init__(self, alphabet=Alphabet.BUCKWALTER):
+        if alphabet not in [Alphabet.IPA, Alphabet.BUCKWALTER]:
+            raise ValueError("unsupported alphabet")
+        super().__init__(alphabet)
 
     @classmethod
     def get_lang(cls, target_lang: str) -> str:
@@ -26,17 +30,69 @@ class MantoqPhonemizer(BasePhonemizer):
 
     def phonemize_string(self, text: str, lang: str = "ar") -> str:
         """
+        Phonemizes an Arabic string using the Mantoq G2P tool.
+        If the alphabet is set to IPA, it then converts the result using bw2ipa.
         """
         lang = self.get_lang(lang)
+        # The mantoq function returns a tuple of (normalized_text, phonemes)
         normalized_text, phonemes = mantoq(text)
-        return "".join(phonemes).replace("_+_", " ")
+
+        # The phonemes are a list of characters, we join them into a string
+        # and replace the word separator token with a space.
+        phonemes = "".join(phonemes).replace("_+_", " ")
+
+        if self.alphabet == Alphabet.IPA:
+            # If the alphabet is IPA, we use the bw2ipa function to translate
+            # the Buckwalter-like phonemes into IPA.
+            return bw2ipa(phonemes)
+
+        # Otherwise, we return the phonemes in the default Mantoq alphabet.
+        return phonemes
 
 
 if __name__ == "__main__":
+    from phoonnx.phonemizers.mul import EspeakPhonemizer
+
+    espeak = EspeakPhonemizer()
+
+    # Initialize phonemizers for both MANTOQ and IPA alphabets
+    pho_mantoq = MantoqPhonemizer(alphabet=Alphabet.IPA)
+
+
+    def compare(text):
+        print(f"Original Text: {text}")
+        print(f"  Mantoq: {pho_mantoq.phonemize_string(text, 'ar')}")
+        print(f"  Espeak: {espeak.phonemize_string(text, 'ar')}")
+
+        ts = pho_mantoq.add_diacritics(text, 'ar')
+        print(f"Tashkeel Text: {ts}")
+        print(f"  Mantoq: {pho_mantoq.phonemize_string(ts, 'ar')}")
+        print(f"  Espeak: {espeak.phonemize_string(ts, 'ar')}")
+        print("\n#########################")
+
+
     text = "مرحبا بالعالم"
-    # gets normalized to
-    # مَرْحَبًا بِالْعالَم
+    compare(text)
+
+    text = "ذهب الطالب إلى المكتبة لقراءة كتاب عن تاريخ الأندلس."
+    compare(text)
+
+    # 1. Test for gemination of a sun letter (e.g., ash-shams)
+    text = "الشمس"
+    compare(text)
+
+    # 2. Test for long vowels (e.g., 'fil' - elephant)
+    text = "فيل"
+    compare(text)
+
+    # 3. Test for glide (e.g., 'yawm' - day)
+    text = "يوم"
+    compare(text)
+
+    # 4. Test for long vowels (e.g., 'suwr' - wall)
+    text = "سور"
+    compare(text)
 
-    pho = MantoqPhonemizer()
-    print(pho.phonemize(text, "ar"))
-    # [('m a r H a b a n aa   b i l E aa l a m', '.', True)]
+    # 5. Test for glide (e.g., 'law' - if)
+    text = "لو"
+    compare(text)

phoonnx/phonemizers/base.py

@@ -8,6 +8,8 @@ from langcodes import tag_distance
 from quebra_frases import sentence_tokenize
 from phoonnx.config import Alphabet
 from phoonnx.util import normalize
+from phoonnx.thirdparty.phonikud import PhonikudDiacritizer
+from phoonnx.thirdparty.tashkeel import TashkeelDiacritizer
 
 # list of (substring, terminator, end_of_sentence) tuples.
 TextChunks = List[Tuple[str, str, bool]]
@@ -18,10 +20,27 @@ PhonemizedChunks = list[list[str]]
 
 
 class BasePhonemizer(metaclass=abc.ABCMeta):
-    def __init__(self, alphabet: Alphabet = Alphabet.UNICODE):
+    def __init__(self, alphabet: Alphabet = Alphabet.UNICODE,
+                 taskeen_threshold: Optional[float] = 0.8):
         super().__init__()
         self.alphabet = alphabet
 
+        self.taskeen_threshold = taskeen_threshold  # arabic only
+        self._tashkeel: Optional[TashkeelDiacritizer] = None
+        self._phonikud: Optional[PhonikudDiacritizer] = None # hebrew only
+
+    @property
+    def phonikud(self) -> PhonikudDiacritizer:
+        if self._phonikud is None:
+            self._phonikud = PhonikudDiacritizer()
+        return self._phonikud
+
+    @property
+    def tashkeel(self) -> TashkeelDiacritizer:
+        if self._tashkeel is None:
+            self._tashkeel = TashkeelDiacritizer()
+        return self._tashkeel
+
     @abc.abstractmethod
     def phonemize_string(self, text: str, lang: str) -> str:
         raise NotImplementedError
@@ -29,6 +48,13 @@ class BasePhonemizer(metaclass=abc.ABCMeta):
     def phonemize_to_list(self, text: str, lang: str) -> List[str]:
         return list(self.phonemize_string(text, lang))
 
+    def add_diacritics(self, text: str, lang: str) -> str:
+        if lang.startswith("he"):
+            return self.phonikud.diacritize(text)
+        elif lang.startswith("ar"):
+            return self.tashkeel.diacritize(text, self.taskeen_threshold)
+        return text
+
     def phonemize(self, text: str, lang: str) -> PhonemizedChunks:
         if not text:
             return [('', '', True)]

phoonnx/phonemizers/gl.py

@@ -12,6 +12,57 @@ class CotoviaError(Exception):
     pass
 
 
+COTOVIA2IPA = {
+    "pau": " ",
+    "a": "a",
+    "E": "ɛ",
+    "e": "e",
+    "i": "i",
+    "j": "j",
+    "O": "ɔ",
+    "o": "o",
+    "u": "u",
+    "w": "w",
+    "p": "p",
+    "b": "b",
+    "B": "β",
+    "t": "t",
+    "d": "d",
+    "D": "ð",
+    "k": "k",
+    "g": "g",
+    "G": "ɣ",
+    "f": "f",
+    "T": "θ",
+    "s": "s",
+    "S": "ʃ",
+    "tS": "tʃ",
+    "m": "m",
+    "n": "n",
+    "J": "ɲ",
+    "N": "ŋ",
+    "l": "l",
+    "Z": "ʎ",
+    "jj": "ʎ",
+    "L": "ʎ",
+    "r": "ɾ",
+    "rr": "r",
+    "X": "x"
+}
+
+
+def cotovia2ipa(text: str) -> str:
+    """
+    Converts a string of Cotovía phonemes to IPA.
+    """
+    # Sort the dictionary keys by length in descending order to handle multi-character phonemes first
+    sorted_cotovia_keys = sorted(COTOVIA2IPA.keys(), key=len, reverse=True)
+    ipa_str = text
+    for cotovia_char in sorted_cotovia_keys:
+        ipa_str = ipa_str.replace(cotovia_char, COTOVIA2IPA[cotovia_char])
+    return ipa_str
+
+
 class CotoviaPhonemizer(BasePhonemizer):
     """
     A phonemizer class that uses the Cotovia TTS binary to convert text into phonemes.
@@ -19,7 +70,7 @@ class CotoviaPhonemizer(BasePhonemizer):
     regular expression transformations to clean and normalize the phonetic representation.
     """
 
-    def __init__(self, cotovia_bin_path: Optional[str] = None):
+    def __init__(self, cotovia_bin_path: Optional[str] = None, alphabet: Alphabet = Alphabet.IPA):
         """
         Initializes the CotoviaPhonemizer.
 
@@ -31,7 +82,7 @@ class CotoviaPhonemizer(BasePhonemizer):
         if not os.path.exists(self.cotovia_bin):
             raise FileNotFoundError(f"Cotovia binary not found at {self.cotovia_bin}. "
                                     "Please ensure it's installed or provide the correct path.")
-        super().__init__(Alphabet.COTOVIA)
+        super().__init__(alphabet)
 
     @classmethod
     def get_lang(cls, target_lang: str) -> str:
@@ -127,6 +178,8 @@ class CotoviaPhonemizer(BasePhonemizer):
         # substitute ' ( text )' to ', text,'
         str_ext = re.sub(r"(\w+)\s*\(\s*([^\(\)]*?)\s*\)", r"\1, \\2,", str_ext)
 
+        if self.alphabet == Alphabet.IPA:
+            return cotovia2ipa(str_ext)
         return str_ext
 
 
@@ -138,5 +191,5 @@ if __name__ == "__main__":
     lang = "gl"
     text_gl = "Este é un sistema de conversión de texto a voz en lingua galega baseado en redes neuronais artificiais. Ten en conta que as funcionalidades incluídas nesta páxina ofrécense unicamente con fins de demostración. Se tes algún comentario, suxestión ou detectas algún problema durante a demostración, ponte en contacto connosco."
     print(f"\n--- Getting phonemes for '{text_gl}' (Cotovia) ---")
-    phonemes_cotovia = cotovia.phonemize(text_gl, lang)
+    phonemes_cotovia = cotovia.phonemize_string(text_gl, lang)
     print(f"  Cotovia Phonemes: {phonemes_cotovia}")

phoonnx/phonemizers/he.py

@@ -1,30 +1,12 @@
-import os.path
-
-import requests
-
-from phoonnx.phonemizers.base import BasePhonemizer
 from phoonnx.config import Alphabet
+from phoonnx.phonemizers.base import BasePhonemizer
 
 
 class PhonikudPhonemizer(BasePhonemizer):
-    dl_url = "https://huggingface.co/thewh1teagle/phonikud-onnx/resolve/main/phonikud-1.0.int8.onnx"
 
-    def __init__(self, model: str = None, diacritics=True):
-        from phonikud_onnx import Phonikud
+    def __init__(self):
         from phonikud import phonemize
         self.g2p = phonemize
-        self.diacritics = diacritics
-        if model is None:
-            base_path = os.path.expanduser("~/.local/share/phonikud")
-            fname = self.dl_url.split("/")[-1]
-            model = f"{base_path}/{fname}"
-            if not os.path.isfile(model):
-                os.makedirs(base_path, exist_ok=True)
-                # TODO - streaming download
-                data = requests.get(self.dl_url).content
-                with open(model, "wb") as f:
-                    f.write(data)
-        self.phonikud = Phonikud(model) if diacritics else None
         super().__init__(Alphabet.IPA)
 
     @classmethod
@@ -48,20 +30,19 @@ class PhonikudPhonemizer(BasePhonemizer):
         """
         """
         lang = self.get_lang(lang)
-        if self.diacritics:
-            text = self.phonikud.add_diacritics(text)
         return self.g2p(text)
 
 
 if __name__ == "__main__":
-    #text = "מתכת יקרה"
+    # text = "מתכת יקרה"
     text = 'שָׁלוֹם עוֹלָם'
 
-    pho = PhonikudPhonemizer(diacritics=False)
+    pho = PhonikudPhonemizer()
     lang = "he"
 
     print(f"\n--- Getting phonemes for '{text}' ---")
+    # text = pho.add_diacritics(text, lang)
     phonemes = pho.phonemize(text, lang)
     print(f"  Phonemes: {phonemes}")
     # --- Getting phonemes for 'שָׁלוֹם עוֹלָם' ---
-    #   Phonemes: [('ʃalˈom ʔolˈam', '.', True)]
+    #   Phonemes: [('ʃalˈom ʔolˈam', '.', True)]