phoonnx 0.0.0__py3-none-any.whl

phoonnx/phonemizers/zh.py

@@ -0,0 +1,308 @@
+import abc
+from typing import List
+
+
+from phoonnx.phonemizers.base import BasePhonemizer
+from phoonnx.thirdparty.zh_num import num2str
+from phoonnx.config import Alphabet
+
+
+class JiebaPhonemizer(BasePhonemizer):
+    """
+    A non-phonemizing class that simply uses Jieba to segment Chinese text
+    into words with spaces for token separation.
+    """
+    def __init__(self):
+        super().__init__(Alphabet.HANZI)
+
+    @classmethod
+    def get_lang(cls, target_lang: str) -> str:
+        """
+        Validates and returns the closest supported language code.
+
+        Args:
+            target_lang (str): The language code to validate.
+
+        Returns:
+            str: The validated language code.
+
+        Raises:
+            ValueError: If the language code is unsupported.
+        """
+        # this check is here only to throw an exception if invalid language is provided
+        return cls.match_lang(target_lang, ["zh"])
+
+    def phonemize_string(self, text: str, lang: str = "zh") -> str:
+        """
+        Segments the input Chinese text using Jieba.
+
+        Args:
+            text (str): The input sentence.
+            lang (str): Language code (must be "zh").
+
+        Returns:
+            str: Tokenized text with words separated by spaces.
+        """
+        import jieba
+        lang = self.get_lang(lang)
+        seg_list = jieba.cut(text, cut_all=False)
+        seg_list = [num2str(w) if w.isdigit() else w for w in seg_list]
+        return " ".join(seg_list)
+
+
+class BaseChinesePinyinPhonemizer(BasePhonemizer):
+    """
+    Base class for Chinese phonemizers using different pinyin G2P libraries.
+    Supports optional IPA conversion and segmentation via Jieba.
+    """
+
+    def __init__(self, alphabet=Alphabet.PINYIN, jieba: bool = True, retone=True):
+        """
+        Initializes the phonemizer.
+
+        Args:
+            ipa (bool): Whether to convert pinyin to IPA.
+            jieba (bool): Whether to segment text using Jieba before phonemization.
+        """
+        assert alphabet in [Alphabet.PINYIN, Alphabet.IPA]
+        super().__init__(alphabet)
+        self.jieba = jieba
+        self.retone = retone
+        from pinyin_to_ipa import pinyin_to_ipa
+        self.pinyin_to_ipa = pinyin_to_ipa
+
+    @classmethod
+    def get_lang(cls, target_lang: str) -> str:
+        """
+        Validates and returns the closest supported language code.
+
+        Args:
+            target_lang (str): The language code to validate.
+
+        Returns:
+            str: The validated language code.
+
+        Raises:
+            ValueError: If the language code is unsupported.
+        """
+        # this check is here only to throw an exception if invalid language is provided
+        return cls.match_lang(target_lang, ["zh"])
+
+    @staticmethod
+    def _retone(p):
+        p = p.replace('˧˩˧', '↓')  # third tone
+        p = p.replace('˧˥', '↗')  # second tone
+        p = p.replace('˥˩', '↘')  # fourth tone
+        p = p.replace('˥', '→')  # first tone
+        p = p.replace(chr(635) + chr(809), 'ɨ').replace(chr(633) + chr(809), 'ɨ')
+        assert chr(809) not in p, p
+        return p
+
+    def to_ipa(self, phones: List[str]) -> List[str]:
+        """
+        Converts a list of pinyin syllables to IPA. Falls back to the original syllable if conversion fails.
+
+        Args:
+            phones (List[str]): List of pinyin syllables or phrases.
+
+        Returns:
+            List[str]: Corresponding IPA or original syllables.
+        """
+        ipa_phones: List[str] = []
+        for p in phones:
+            if p == " ":
+                ipa_phones.append(" ")
+                continue
+            pho_str = ""
+            for sp in p.split():  # G2P might return phrases with multiple syllables
+                try:
+                    pho = self.pinyin_to_ipa(sp.strip())[0][0]
+                    if self.retone:
+                        pho = self._retone(pho)
+                    pho_str += pho
+                except Exception:
+                    pass
+            ipa_phones.append(pho_str)
+        return ipa_phones
+
+    def phonemize_to_list(self, text: str, lang: str) -> List[str]:
+        phones: List[str] = []
+        lang = self.get_lang(lang)
+        if self.jieba:
+            import jieba
+            for chunk in jieba.cut(text, cut_all=False):
+                if chunk.isdigit():
+                    chunk = num2str(chunk)
+                phones += self.get_pinyin(chunk)
+                phones += [" "]  # keep jieba whitespace
+        else:
+            phones = self.get_pinyin(text)
+        if self.alphabet == Alphabet.IPA:
+            phones = self.to_ipa(phones)
+        return phones
+
+    def phonemize_string(self, text: str, lang: str = "zh") -> str:
+        """
+        Converts input text to a space-separated phoneme string.
+
+        Args:
+            text (str): The input sentence.
+            lang (str): Language code (must be "zh").
+
+        Returns:
+            str: Space-separated phoneme string.
+        """
+        phones: List[str] = self.phonemize_to_list(text, lang)
+        return "".join(phones)
+
+    @abc.abstractmethod
+    def get_pinyin(self, text: str) -> List[str]:
+        """
+        Abstract method to be implemented by subclasses for converting text to pinyin.
+
+        Args:
+            text (str): Input Chinese text.
+
+        Returns:
+            List[str]: List of pinyin tokens.
+        """
+        return NotImplemented
+
+
+class G2pCPhonemizer(BaseChinesePinyinPhonemizer):
+    """
+    Phonemizer using g2pc (CRF-based Grapheme-to-Phoneme converter).
+    https://github.com/Kyubyong/g2pC
+    """
+
+    def __init__(self, alphabet=Alphabet.PINYIN, jieba: bool = True):
+        from g2pc import G2pC
+        self.g2p = G2pC()
+        super().__init__(alphabet, jieba)
+
+    def get_pinyin(self, text: str) -> List[str]:
+        """
+        Returns a list of pinyin syllables from g2pc.
+
+        Args:
+            text (str): Input Chinese text.
+
+        Returns:
+            List[str]: Pinyin tokens.
+        """
+        return [a[3] for a in self.g2p(text)]
+
+
+class G2pMPhonemizer(BaseChinesePinyinPhonemizer):
+    """
+    Phonemizer using g2pM - A Neural Grapheme-to-Phoneme Conversion Package for Mandarin Chinese
+    https://github.com/kakaobrain/g2pm
+    """
+
+    def __init__(self, alphabet=Alphabet.PINYIN, tone: bool = True, char_split: bool = False, jieba: bool = True):
+        from g2pM import G2pM
+        self.g2p = G2pM()
+        self.tone = tone
+        self.char_split = char_split
+        super().__init__(alphabet, jieba)
+
+    def get_pinyin(self, text: str) -> List[str]:
+        """
+        Returns a list of pinyin tokens from g2pM.
+
+        Args:
+            text (str): Input Chinese text.
+
+        Returns:
+            List[str]: Pinyin tokens.
+        """
+        return self.g2p(text, tone=self.tone, char_split=self.char_split)
+
+
+class XpinyinPhonemizer(BaseChinesePinyinPhonemizer):
+    """
+    Phonemizer using xpinyin (basic pinyin generator with optional tone marks).
+    """
+
+    def __init__(self, alphabet=Alphabet.PINYIN, tone_marks: str = "numbers", jieba: bool = True):
+        from xpinyin import Pinyin
+        self.g2p = Pinyin()
+        self.tone_marks = tone_marks
+        super().__init__(alphabet, jieba)
+
+    def get_pinyin(self, text: str) -> List[str]:
+        """
+        Returns a list of pinyin tokens from xpinyin.
+
+        Args:
+            text (str): Input Chinese text.
+
+        Returns:
+            List[str]: Pinyin tokens.
+        """
+        return self.g2p.get_pinyin(text, tone_marks=self.tone_marks).split("-")
+
+
+class PypinyinPhonemizer(BaseChinesePinyinPhonemizer):
+    """
+    Phonemizer using pypinyin (comprehensive and accurate pinyin library).
+    """
+
+    def __init__(self, alphabet=Alphabet.PINYIN, jieba: bool = True):
+        from pypinyin import pinyin
+        self.g2p = pinyin
+        super().__init__(alphabet, jieba)
+
+    def get_pinyin(self, text: str) -> List[str]:
+        """
+        Returns a list of pinyin tokens from pypinyin.
+
+        Args:
+            text (str): Input Chinese text.
+
+        Returns:
+            List[str]: Pinyin tokens.
+        """
+        return [p[0] for p in self.g2p(text)]
+
+
+if __name__ == "__main__":
+    lang = "zh"
+    text = "然而，他红了20年以后，他竟退出了大家的视线。"
+
+    pho = JiebaPhonemizer()
+    #pho1 = G2pCPhonemizer(ipa=True)
+    pho2 = G2pMPhonemizer()
+    pho3 = XpinyinPhonemizer()
+    pho4 = PypinyinPhonemizer()
+
+    from phoonnx.phonemizers.mul import MisakiPhonemizer
+
+    pho5 = MisakiPhonemizer()
+
+    print(f"\n--- Getting phonemes for '{text}' ---")
+
+    phones = pho5.phonemize_to_list(text, lang)
+    print(f" Misaki: {phones}")
+
+    #phones = pho1.phonemize(text, lang)
+    #print(f" G2pC: {phones}")
+
+    phones = pho2.phonemize_to_list(text, lang)
+    print(f" G2pM: {phones}")
+
+    phones = pho3.phonemize_to_list(text, lang)
+    print(f" Xpinyin: {phones}")
+
+    phones = pho4.phonemize_to_list(text, lang)
+    print(f" Pypinyin: {phones}")
+
+    phones = pho.phonemize_to_list(text, lang)
+    print(f" Jieba: {phones}")
+
+    #
+    exit()
+    #   Phonemes: [('ran2 er2 ， ta1 hong2 le5 2 0 nian2 yi3 hou4 ， ta1 jing4 tui4 chu1 le5 da4 jia1 de5 shi4 xian4 。', '.', True)]
+    #   Phonemes: [('ran2 er2 ， ta1 hong2 le5 20 nian2 yi3 hou4 ， ta1 jing4 tui4 chu1 le5 da4 jia1 de5 shi4 xian4 。', '.', True)]
+    #   Phonemes: [('ran2 er2 ， ta1 hong2 le5 20 nian2 yi3 hou4 ， ta1 jing4 tui4 chu1 le5 da4 jia1 de5 shi4 xian4 。', '.', True)]
+    #   Phonemes: [('rán ér ， tā hóng le 20 nián yǐ hòu ， tā jìng tuì chū le dà jiā de shì xiàn 。', '.', True)]

phoonnx/thirdparty/arpa2ipa.py

@@ -0,0 +1,249 @@
+# taken from https://github.com/chorusai/arpa2ipa/blob/master/arpa2ipa/_arpa_to_ipa.py
+import re
+
+"""
+https://en.wikipedia.org/wiki/Arpabet
+
+In Arpabet, every phoneme is represented by one or two capital letters. 
+Digits are used as stress indicators and are placed at the end of the 
+stressed syllabic vowel. Punctuation marks are used like in the written 
+language, to represent intonation changes at the end of clauses and 
+sentences. The stress values are:
+
+Value | Description
+    0 | No stress
+    1 | Primary stress
+    2 | Seconary stress
+
+** Vowels
+*** Monophthongs
+
+Arpabet  IPA Word examples
+     AO    ɔ off (AO1 F); fall (F AO1 L); frost (F R AO1 S T)
+     AA    ɑ father (F AA1 DH ER), cot (K AA1 T)
+     IY    i bee (B IY1); she (SH IY1)
+     UW    u you (Y UW1); new (N UW1); food (F UW1 D)
+     EH    ɛ red (R EH1 D); men (M EH1 N) (modern versions use e)
+     IH    ɪ big (B IH1 G); win (W IH1 N)
+     UH    ʊ should (SH UH1 D), could (K UH1 D)
+     AH    ʌ but (B AH1 T), sun (S AH1 N)
+     AH(0) ə sofa (S OW1 F AH0), alone (AH0 L OW1 N)
+     AX    ə discus (D IH1 S K AX0 S);
+     AE    æ at (AE1 T); fast (F AE1 S T)
+*** Dipthongs
+
+Arpabet	 IPA Word Examples
+     EY   eɪ say (S EY1); eight (EY1 T)
+     AY   aɪ my (M AY1); why (W AY1); ride (R AY1 D)
+     OW   oʊ show (SH OW1); coat (K OW1 T)
+     AW    a how (HH AW1); now (N AW1)
+     OY   ɔɪ boy (B OY1); toy (T OY1)
+*** R-colored vowels
+
+     Arpabet IPA Word Examples
+        ER     ɝ her (HH ER0); bird (B ER1 D); hurt (HH ER1 T), nurse (N ER1 S)
+        AXR    ɚ father (F AA1 DH AXR); coward (K AW1 AXR D)
+        EH R  ɛr air (EH1 R); where (W EH1 R); hair (HH EH1 R)
+        UH R  ʊr cure (K Y UH1 R); bureau (B Y UH1 R OW0), detour (D IH0 T 
+        UH1 R)
+        AO R  ɔr more (M AO1 R); bored (B AO1 R D); chord (K AO1 R D)
+        AA R  ɑr large (L AA1 R JH); hard (HH AA1 R D)
+IH R or IY R  ɪr ear (IY1 R); near (N IH1 R)
+        AW R aʊr This seems to be a rarely used r-controlled vowel. In some 
+        dialects flower (F L AW1 R; in other dialects F L AW1 ER0)
+** Consonants
+*** Stops
+Arpabet	IPA	Word Examples
+      P   p pay (P EY1)
+      B   b buy (B AY1)
+      T   t take (T EY1 K)
+      D   d day (D EY1)
+      K   k key (K IY1)
+      G   ɡ go (G OW1)
+
+*** Affricates
+Arpabet IPA Word Examples
+     CH  tʃ chair (CH EH1 R)
+     JH  dʒ just (JH AH1 S T); gym (JH IH1 M)
+
+*** Fricatives
+Arpabet IPA Word Examples
+      F   f for (F AO1 R)
+      V   v very (V EH1 R IY0)
+     TH   θ thanks (TH AE1 NG K S); Thursday (TH ER1 Z D EY2)
+     DH   ð that (DH AE1 T); the (DH AH0); them (DH EH1 M)
+      S   s say (S EY1)
+      Z   z zoo (Z UW1)
+     SH   ʃ show (SH OW1)
+     ZH   ʒ measure (M EH1 ZH ER0); pleasure (P L EH1 ZH ER)
+     HH   h house (HH AW1 S)
+
+*** Nasals
+Arpabet IPA Word Examples
+      M   m man (M AE1 N)
+     EM   m̩ keep 'em (K IY1 P EM)
+      N   n no (N OW1)
+     EN   n̩ button (B AH1 T EN)
+     NG   ŋ sing (S IH1 NG)
+    ENG   ŋ̍ Washington (W AO1 SH ENG T EN)
+
+*** Liquids
+Arpabet    IPA Word Examples
+      L	l or ɫ late (L EY1 T); fail (F EY1 L)
+     EL      ɫ̩ bottle (B AO1 DX EL)
+      R r or ɹ run (R AH1 N)
+     DX      ɾ wetter (W EH1 DX AXR)
+     NX      ɾ̃ wintergreen (W IY2 NX AXR G R IY1 N)
+
+*** Semivowels
+  Arpabet       IPA Word Examples
+        Y         j yes (Y EH1 S)
+        W         w way (W EY1)
+        Q         ʔ (glottal stop) uh-oh (Q AH1 Q OW) (ʔʌʔoʊ)
+(missing)	hw or ʍ	"when" etc. in some dialects
+
+"""
+monopthongs = {
+    'AO': 'ɔ',
+    'AO0': 'ɔ',
+    'AO1': 'ɔ',
+    'AO2': 'ɔ',
+    'AA': 'ɑ',
+    'AA0': 'ɑ',
+    'AA1': 'ɑ',
+    'AA2': 'ɑ',
+    'IY': 'i',
+    'IY0': 'i',
+    'IY1': 'i',
+    'IY2': 'i',
+    'UW': 'u',
+    'UW0': 'u',
+    'UW1': 'u',
+    'UW2': 'u',
+    'EH': 'e',
+    'EH0': 'e',
+    'EH1': 'e',
+    'EH2': 'e',
+    'IH': 'ɪ',
+    'IH0': 'ɪ',
+    'IH1': 'ɪ',
+    'IH2': 'ɪ',
+    'UH': 'ʊ',
+    'UH0': 'ʊ',
+    'UH1': 'ʊ',
+    'UH2': 'ʊ',
+    'AH': 'ʌ',
+    'AH0': 'ə',
+    'AH1': 'ʌ',
+    'AH2': 'ʌ',
+    'AE': 'æ',
+    'AE0': 'æ',
+    'AE1': 'æ',
+    'AE2': 'æ',
+    'AX': 'ə',
+    'AX0': 'ə',
+    'AX1': 'ə',
+    'AX2': 'ə',
+}
+
+dipthongs = {
+    'EY': 'eɪ',
+    'EY0': 'eɪ',
+    'EY1': 'eɪ',
+    'EY2': 'eɪ',
+    'AY': 'aɪ',
+    'AY0': 'aɪ',
+    'AY1': 'aɪ',
+    'AY2': 'aɪ',
+    'OW': 'oʊ',
+    'OW0': 'oʊ',
+    'OW1': 'oʊ',
+    'OW2': 'oʊ',
+    'AW': 'aʊ',
+    'AW0': 'aʊ',
+    'AW1': 'aʊ',
+    'AW2': 'aʊ',
+    'OY': 'ɔɪ',
+    'OY0': 'ɔɪ',
+    'OY1': 'ɔɪ',
+    'OY2': 'ɔɪ',
+}
+
+r_colored_vowels = {
+    'ER': 'ɜr',
+    'ER0': 'ɜr',
+    'ER1': 'ɜr',
+    'ER2': 'ɜr',
+    'AXR': 'ər',
+    'AXR0': 'ər',
+    'AXR1': 'ər',
+    'AXR2': 'ər',
+}
+
+stops = {
+    'P': 'p',
+    'B': 'b',
+    'T': 't',
+    'D': 'd',
+    'K': 'k',
+    'G': 'g',
+}
+
+affricates = {
+    'CH': 'tʃ',
+    'JH': 'dʒ',
+}
+
+fricatives = {
+    'F': 'f',
+    'V': 'v',
+    'TH': 'θ',
+    'DH': 'ð',
+    'S': 's',
+    'Z': 'z',
+    'SH': 'ʃ',
+    'ZH': 'ʒ',
+    'HH': 'h',
+}
+
+nasals = {
+    'M': 'm',
+    'EM': 'm̩',
+    'N': 'n',
+    'EN': 'n̩',
+    'NG': 'ŋ',
+    'ENG': 'ŋ̍',
+}
+
+liquids = {
+    'L': 'l',
+    'EL': 'ɫ̩',
+    'R': 'r',
+    'DX': 'ɾ',
+    'NX': 'ɾ̃',
+}
+
+semivowels = {
+    'W': 'w',
+    'Y': 'j',
+    'Q': 'ʔ'
+}
+
+arpa_to_ipa_lookup = {}
+arpa_to_ipa_lookup.update(monopthongs)
+arpa_to_ipa_lookup.update(dipthongs)
+arpa_to_ipa_lookup.update(r_colored_vowels)
+arpa_to_ipa_lookup.update(stops)
+arpa_to_ipa_lookup.update(affricates)
+arpa_to_ipa_lookup.update(fricatives)
+arpa_to_ipa_lookup.update(nasals)
+arpa_to_ipa_lookup.update(liquids)
+arpa_to_ipa_lookup.update(semivowels)
+
+
+def split_on_capital(camel):
+    ' '.join(re.findall('[A-Z][a-z]*', camel.title())).lower()
+
+
+def arpa_to_ipa(arpa):
+    return ' '.join(arpa_to_ipa_lookup[phoneme] for phoneme in arpa.split(' '))