phoonnx 0.0.2a2__tar.gz → 0.1.0a1__tar.gz

{phoonnx-0.0.2a2 → phoonnx-0.1.0a1}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: phoonnx
-Version: 0.0.2a2
+Version: 0.1.0a1
 Home-page: https://github.com/TigreGotico/phoonnx
 Author: JarbasAi
 Author-email: jarbasai@mailfence.com

phoonnx-0.1.0a1/phoonnx/phonemizers/ar.py

@@ -0,0 +1,106 @@
+from phoonnx.phonemizers.base import BasePhonemizer
+from phoonnx.thirdparty.mantoq import g2p as mantoq
+from phoonnx.config import Alphabet
+from phoonnx.thirdparty.bw2ipa import translate as bw2ipa
+
+
+class MantoqPhonemizer(BasePhonemizer):
+
+    def __init__(self, alphabet=Alphabet.MANTOQ):
+        if alphabet not in [Alphabet.IPA, Alphabet.MANTOQ]:
+            raise ValueError("unsupported alphabet")
+        super().__init__(alphabet)
+
+    @classmethod
+    def get_lang(cls, target_lang: str) -> str:
+        """
+        Validates and returns the closest supported language code.
+
+        Args:
+            target_lang (str): The language code to validate.
+
+        Returns:
+            str: The validated language code.
+
+        Raises:
+            ValueError: If the language code is unsupported.
+        """
+        # this check is here only to throw an exception if invalid language is provided
+        return cls.match_lang(target_lang, ["ar"])
+
+    def phonemize_string(self, text: str, lang: str = "ar") -> str:
+        """
+        Phonemizes an Arabic string using the Mantoq G2P tool.
+        If the alphabet is set to IPA, it then converts the result using bw2ipa.
+        """
+        lang = self.get_lang(lang)
+        # The mantoq function returns a tuple of (normalized_text, phonemes)
+        normalized_text, phonemes = mantoq(text)
+
+        # The phonemes are a list of characters, we join them into a string
+        # and replace the word separator token with a space.
+        phonemes =  "".join(phonemes).replace("_+_", " ")
+
+        if self.alphabet == Alphabet.IPA:
+            # If the alphabet is IPA, we use the bw2ipa function to translate
+            # the Buckwalter-like phonemes into IPA.
+            return bw2ipa(phonemes)
+
+        # Otherwise, we return the phonemes in the default Mantoq alphabet.
+        return phonemes
+
+
+if __name__ == "__main__":
+    # Initialize phonemizers for both MANTOQ and IPA alphabets
+    pho_mantoq = MantoqPhonemizer()
+    pho_ipa = MantoqPhonemizer(alphabet=Alphabet.IPA)
+
+    text1 = "مرحبا بالعالم"
+    print(f"Original Text: {text1}")
+    print(f"  Mantoq Phonemizer: {pho_mantoq.phonemize_string(text1, 'ar')}")
+    print(f"  IPA Phonemizer:    {pho_ipa.phonemize_string(text1, 'ar')}")
+    print("-" * 20)
+
+    text2 = "ذهب الطالب إلى المكتبة لقراءة كتاب عن تاريخ الأندلس."
+    print(f"Original Text: {text2}")
+    print(f"  Mantoq Phonemizer: {pho_mantoq.phonemize_string(text2, 'ar')}")
+    print(f"  IPA Phonemizer:    {pho_ipa.phonemize_string(text2, 'ar')}")
+    print("-" * 20)
+
+    # --- New Test Cases to check bw2ipa logic ---
+    print("--- New Test Cases for bw2ipa logic ---")
+
+    # 1. Test for gemination of a sun letter (e.g., ash-shams)
+    text3 = "الشمس"
+    print(f"Original Text: '{text3}'")
+    print(f"  Mantoq Phonemizer: {pho_mantoq.phonemize_string(text3, 'ar')}")
+    print(f"  IPA Phonemizer:    {pho_ipa.phonemize_string(text3, 'ar')}")
+    print("-" * 20)
+
+    # 2. Test for long vowels (e.g., 'fil' - elephant)
+    text4 = "فيل"
+    print(f"Original Text: '{text4}'")
+    print(f"  Mantoq Phonemizer: {pho_mantoq.phonemize_string(text4, 'ar')}")
+    print(f"  IPA Phonemizer:    {pho_ipa.phonemize_string(text4, 'ar')}")
+    print("-" * 20)
+
+    # 3. Test for glide (e.g., 'yawm' - day)
+    text5 = "يوم"
+    print(f"Original Text: '{text5}'")
+    print(f"  Mantoq Phonemizer: {pho_mantoq.phonemize_string(text5, 'ar')}")
+    print(f"  IPA Phonemizer:    {pho_ipa.phonemize_string(text5, 'ar')}")
+    print("-" * 20)
+
+    # 4. Test for long vowels (e.g., 'suwr' - wall)
+    text6 = "سور"
+    print(f"Original Text: '{text6}'")
+    print(f"  Mantoq Phonemizer: {pho_mantoq.phonemize_string(text6, 'ar')}")
+    print(f"  IPA Phonemizer:    {pho_ipa.phonemize_string(text6, 'ar')}")
+    print("-" * 20)
+
+    # 5. Test for glide (e.g., 'law' - if)
+    text7 = "لو"
+    print(f"Original Text: '{text7}'")
+    print(f"  Mantoq Phonemizer: {pho_mantoq.phonemize_string(text7, 'ar')}")
+    print(f"  IPA Phonemizer:    {pho_ipa.phonemize_string(text7, 'ar')}")
+    print("-" * 20)

phoonnx-0.1.0a1/phoonnx/thirdparty/bw2ipa.py

@@ -0,0 +1,66 @@
+# -*- coding: UTF8 -*-
+"""
+This script translates Buckwalter-transcribed Modern Standard Arabic to IPA.
+
+This relies on Mantoq tokenization that uses separate tokens for vowel length and consonant gemination.
+
+by Casimiro Ferreira with help of Gemini July 2025.
+"""
+
+import re
+
+# This dictionary maps a single Buckwalter character to its most common IPA equivalent.
+char_dict = {
+    'a': 'a', 'A': 'aː', 'b': 'b', 'c': 'x', 'd': 'd', 'D': 'dˤ', 'e': 'e', 'E': 'ʕ',
+    'f': 'f', 'g': 'ɣ', 'h': 'h', 'H': 'ħ', 'i': 'i', 'I': 'iː', 'j': 'ʒ', 'k': 'k',
+    'l': 'l', 'm': 'm', 'n': 'n', 'p': 'p', 'q': 'q', 'r': 'r', 'R': 'r', 's': 's',
+    'S': 'sˤ', 't': 't', 'T': 'tˤ', 'u': 'u', 'U': 'uː', 'v': 'v', 'w': 'w', 'x': 'x',
+    'y': 'j', 'z': 'z', 'Z': 'ðˤ', '\'': 'ʔ', '<': 'ʔ', 'o': 'o', '-': ' ',
+    '*': 'ð', '$': 'ʃ'
+}
+_vowels = {'a', 'i', 'u', 'aː', 'iː', 'uː'}
+
+
+def translate(buckwalter_text: str) -> str:
+    """
+    Translates a Buckwalter-transcribed string into an IPA string.
+
+    Args:
+        buckwalter_text (str): The Buckwalter string to translate.
+
+    Returns:
+        str: The translated IPA string.
+    """
+    ipa_list = []
+    i = 0
+    while i < len(buckwalter_text):
+        # Check for the longest token first. The new Mantoq tokenization
+        # seems to use a 5-character token.
+        token = buckwalter_text[i:i + 5]
+
+        # If the previous character was a vowel, we assume it's a long vowel
+        # marker (ː). Otherwise, we assume it's a geminated consonant.
+        if token == '_dbl_':
+            if ipa_list and ipa_list[-1] in _vowels:
+                ipa_list.append('ː')  # Add length marker for long vowels
+            elif ipa_list:
+                ipa_list.append(ipa_list[-1])  # Duplicate the consonant
+            i += 5
+            continue
+
+        # Check for multi-character mappings from char_dict
+        two_char_token = buckwalter_text[i:i + 2]
+        if two_char_token in char_dict:
+            ipa_list.append(char_dict[two_char_token])
+            i += 2
+            continue
+
+        # Handle single characters
+        single_char = buckwalter_text[i]
+        if single_char in char_dict:
+            ipa_list.append(char_dict[single_char])
+        else:
+            ipa_list.append(single_char)
+        i += 1
+
+    return ''.join(ipa_list)

{phoonnx-0.0.2a2 → phoonnx-0.1.0a1}/phoonnx/version.py

@@ -1,6 +1,6 @@
 # START_VERSION_BLOCK
 VERSION_MAJOR = 0
-VERSION_MINOR = 0
-VERSION_BUILD = 2
-VERSION_ALPHA = 2
+VERSION_MINOR = 1
+VERSION_BUILD = 0
+VERSION_ALPHA = 1
 # END_VERSION_BLOCK

{phoonnx-0.0.2a2 → phoonnx-0.1.0a1}/phoonnx.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: phoonnx
-Version: 0.0.2a2
+Version: 0.1.0a1
 Home-page: https://github.com/TigreGotico/phoonnx
 Author: JarbasAi
 Author-email: jarbasai@mailfence.com

{phoonnx-0.0.2a2 → phoonnx-0.1.0a1}/phoonnx.egg-info/SOURCES.txt

@@ -29,6 +29,7 @@ phoonnx/phonemizers/vi.py
 phoonnx/phonemizers/zh.py
 phoonnx/thirdparty/__init__.py
 phoonnx/thirdparty/arpa2ipa.py
+phoonnx/thirdparty/bw2ipa.py
 phoonnx/thirdparty/hangul2ipa.py
 phoonnx/thirdparty/zh_num.py
 phoonnx/thirdparty/cotovia/cotovia_aarch64
@@ -93,6 +94,7 @@ phoonnx_train/../phoonnx/phonemizers/vi.py
 phoonnx_train/../phoonnx/phonemizers/zh.py
 phoonnx_train/../phoonnx/thirdparty/__init__.py
 phoonnx_train/../phoonnx/thirdparty/arpa2ipa.py
+phoonnx_train/../phoonnx/thirdparty/bw2ipa.py
 phoonnx_train/../phoonnx/thirdparty/hangul2ipa.py
 phoonnx_train/../phoonnx/thirdparty/zh_num.py
 phoonnx_train/../phoonnx/thirdparty/cotovia/cotovia_aarch64

phoonnx-0.0.2a2/phoonnx/phonemizers/ar.py

@@ -1,42 +0,0 @@
-from phoonnx.phonemizers.base import BasePhonemizer
-from phoonnx.thirdparty.mantoq import g2p as mantoq
-from phoonnx.config import Alphabet
-
-class MantoqPhonemizer(BasePhonemizer):
-
-    def __init__(self):
-        super().__init__(Alphabet.MANTOQ)
-
-    @classmethod
-    def get_lang(cls, target_lang: str) -> str:
-        """
-        Validates and returns the closest supported language code.
-
-        Args:
-            target_lang (str): The language code to validate.
-
-        Returns:
-            str: The validated language code.
-
-        Raises:
-            ValueError: If the language code is unsupported.
-        """
-        # this check is here only to throw an exception if invalid language is provided
-        return cls.match_lang(target_lang, ["ar"])
-
-    def phonemize_string(self, text: str, lang: str = "ar") -> str:
-        """
-        """
-        lang = self.get_lang(lang)
-        normalized_text, phonemes = mantoq(text)
-        return "".join(phonemes).replace("_+_", " ")
-
-
-if __name__ == "__main__":
-    text = "مرحبا بالعالم"
-    # gets normalized to
-    # مَرْحَبًا بِالْعالَم
-
-    pho = MantoqPhonemizer()
-    print(pho.phonemize(text, "ar"))
-    # [('m a r H a b a n aa   b i l E aa l a m', '.', True)]