phoonnx 0.1.0a1__tar.gz → 0.1.1a1__tar.gz

{phoonnx-0.1.0a1 → phoonnx-0.1.1a1}/PKG-INFO

@@ -1,111 +1,112 @@
 Metadata-Version: 2.1
 Name: phoonnx
-Version: 0.1.0a1
+Version: 0.1.1a1
 Home-page: https://github.com/TigreGotico/phoonnx
 Author: JarbasAi
 Author-email: jarbasai@mailfence.com
-Provides-Extra: ceb
-Provides-Extra: ka
-Provides-Extra: cjy
-Provides-Extra: th
-Provides-Extra: ta
-Provides-Extra: ti
+Provides-Extra: cs
+Provides-Extra: mr
+Provides-Extra: tpi
+Provides-Extra: so
 Provides-Extra: uz
-Provides-Extra: ca
+Provides-Extra: de
+Provides-Extra: th
+Provides-Extra: ka
+Provides-Extra: uk
+Provides-Extra: en
+Provides-Extra: uew
+Provides-Extra: kbd
+Provides-Extra: hr
+Provides-Extra: sv
 Provides-Extra: ko
-Provides-Extra: quy
-Provides-Extra: ilo
-Provides-Extra: nan
-Provides-Extra: jv
-Provides-Extra: gan
-Provides-Extra: mr
-Provides-Extra: aa
-Provides-Extra: bn
-Provides-Extra: am
-Provides-Extra: rn
-Provides-Extra: ff
-Provides-Extra: km
-Provides-Extra: ha
-Provides-Extra: lv
-Provides-Extra: si
-Provides-Extra: ar
-Provides-Extra: cs
-Provides-Extra: rw
-Provides-Extra: mi
-Provides-Extra: wuu
-Provides-Extra: nl
-Provides-Extra: hak
-Provides-Extra: mt
+Provides-Extra: got
+Provides-Extra: id
+Provides-Extra: sn
 Provides-Extra: ug
-Provides-Extra: ku
-Provides-Extra: hr
-Provides-Extra: tr
-Provides-Extra: za
-Provides-Extra: csb
-Provides-Extra: te
-Provides-Extra: ur
+Provides-Extra: az
+Provides-Extra: mt
+Provides-Extra: pl
+Provides-Extra: ceb
+Provides-Extra: ky
+Provides-Extra: sq
+Provides-Extra: ny
+Provides-Extra: ti
+Provides-Extra: sg
+Provides-Extra: et
+Provides-Extra: eo
 Provides-Extra: tk
-Provides-Extra: lb
-Provides-Extra: ja
-Provides-Extra: es
-Provides-Extra: lo
-Provides-Extra: id
-Provides-Extra: pa
-Provides-Extra: it
+Provides-Extra: yue
 Provides-Extra: luy
-Provides-Extra: de
-Provides-Extra: sn
-Provides-Extra: fi
-Provides-Extra: gl
-Provides-Extra: sr
-Provides-Extra: so
-Provides-Extra: yo
-Provides-Extra: or
-Provides-Extra: my
-Provides-Extra: xh
-Provides-Extra: fr
+Provides-Extra: jam
+Provides-Extra: hsn
+Provides-Extra: fil
+Provides-Extra: ml
+Provides-Extra: mwl
+Provides-Extra: lez
+Provides-Extra: si
+Provides-Extra: kk
 Provides-Extra: hi
-Provides-Extra: sg
-Provides-Extra: eu
-Provides-Extra: got
-Provides-Extra: tpi
-Provides-Extra: pt
-Provides-Extra: ckb
+Provides-Extra: quy
+Provides-Extra: zh
+Provides-Extra: ru
+Provides-Extra: aii
 Provides-Extra: train
-Provides-Extra: lsm
-Provides-Extra: fa
-Provides-Extra: av
-Provides-Extra: kk
-Provides-Extra: ky
-Provides-Extra: lez
-Provides-Extra: eo
-Provides-Extra: uew
-Provides-Extra: vi
+Provides-Extra: my
+Provides-Extra: yo
+Provides-Extra: cjy
+Provides-Extra: es
 Provides-Extra: he
+Provides-Extra: sr
+Provides-Extra: fr
+Provides-Extra: ro
 Provides-Extra: hmn
-Provides-Extra: hsn
-Provides-Extra: et
 Provides-Extra: lt
-Provides-Extra: ro
+Provides-Extra: ckb
+Provides-Extra: tg
+Provides-Extra: vi
+Provides-Extra: hu
+Provides-Extra: pt
+Provides-Extra: ms
+Provides-Extra: mi
+Provides-Extra: pa
+Provides-Extra: za
+Provides-Extra: lv
+Provides-Extra: am
+Provides-Extra: ca
+Provides-Extra: om
+Provides-Extra: rw
+Provides-Extra: nl
+Provides-Extra: lsm
+Provides-Extra: bn
+Provides-Extra: fa
+Provides-Extra: eu
+Provides-Extra: te
+Provides-Extra: km
 Provides-Extra: lij
+Provides-Extra: tr
+Provides-Extra: ku
+Provides-Extra: lb
+Provides-Extra: ta
+Provides-Extra: av
+Provides-Extra: ha
+Provides-Extra: ff
+Provides-Extra: fi
 Provides-Extra: sw
-Provides-Extra: az
-Provides-Extra: ms
-Provides-Extra: sv
-Provides-Extra: hu
+Provides-Extra: ilo
+Provides-Extra: ja
+Provides-Extra: ar
 Provides-Extra: zu
-Provides-Extra: kbd
-Provides-Extra: jam
-Provides-Extra: sq
-Provides-Extra: tg
-Provides-Extra: ru
-Provides-Extra: ny
-Provides-Extra: yue
-Provides-Extra: zh
-Provides-Extra: uk
-Provides-Extra: om
-Provides-Extra: en
-Provides-Extra: aii
-Provides-Extra: fil
-Provides-Extra: pl
-Provides-Extra: ml
+Provides-Extra: xh
+Provides-Extra: lo
+Provides-Extra: gl
+Provides-Extra: aa
+Provides-Extra: it
+Provides-Extra: ur
+Provides-Extra: hak
+Provides-Extra: wuu
+Provides-Extra: csb
+Provides-Extra: nan
+Provides-Extra: jv
+Provides-Extra: or
+Provides-Extra: gan
+Provides-Extra: rn

phoonnx-0.1.1a1/README.md

@@ -0,0 +1,128 @@
+[![Ask DeepWiki](https://deepwiki.com/badge.svg)](https://deepwiki.com/TigreGotico/phoonnx)
+
+# Phoonnx
+
+A Python library for multilingual phonemization and Text-to-Speech (TTS) using ONNX models.
+
+## Introduction
+
+`phoonnx` is a comprehensive toolkit for performing high-quality, efficient TTS inference using ONNX-compatible models.
+It provides a flexible framework for text normalization, phonemization, and speech synthesis, with built-in support for
+multiple languages and phonemic alphabets. The library is also designed to work with models trained using
+`phoonnx_train`, including utilities for dataset preprocessing and exporting models to the ONNX format.
+
+## Features
+
+- **Efficient Inference:** Leverages `onnxruntime` for fast and efficient TTS synthesis.
+- **Multilingual Support:** Supports a wide range of languages and phonemic alphabets, including IPA, ARPA, Hangul (
+  Korean), and Pinyin (Chinese).
+- **Multiple Phonemizers:** Integrates with various phonemizers like eSpeak, Gruut, and Epitran to convert text to
+  phonemes.
+- **Advanced Text Normalization:** Includes robust utilities for expanding contractions and pronouncing numbers and
+  dates.
+- **Dataset Preprocessing:** Provides a command-line tool to prepare LJSpeech-style datasets for training.
+- **Model Export:** A script is included to convert trained models into the ONNX format, ready for deployment.
+
+## Installation
+
+As `phoonnx` is available on PyPI, you can install it using pip.
+
+```bash
+pip install phoonnx
+```
+
+## Usage
+
+### Synthesizing Speech
+
+The main component for inference is the `TTSVoice` class. You can load a model and synthesize speech from text as
+follows:
+
+```python
+import wave
+
+from phoonnx.config import VoiceConfig, SynthesisConfig
+from phoonnx.voice import TTSVoice
+
+# Load a pre-trained ONNX model and its configuration
+voice = TTSVoice.load("model.onnx", "config.json")
+
+# Configure the synthesis parameters (optional)
+synthesis_config = SynthesisConfig(
+    noise_scale=0.667,
+    length_scale=1.0,
+    noise_w_scale=0.8,
+    enable_phonetic_spellings=True, # apply pronunciation fixes, see "locale" folder in this repo
+    add_diacritics=False  # for arabic and hebrew
+)
+
+# Synthesize audio from text
+text = "Hello, this is a test of the phoonnx library."
+slug = f"phoonnx_{voice.config.phoneme_type.value}_{voice.config.lang_code}"
+with wave.open(f"{slug}.wav", "wb") as wav_file:
+    voice.synthesize_wav(text, wav_file, synthesis_config)
+
+```
+
+### Training
+
+See the dedicated [training.md](/TRAINING.md)
+
+## Supported Phonemizers
+
+`phoonnx` leverages several external Grapheme-to-Phoneme (G2P) and text-processing libraries to provide flexible and
+high-quality phonemization across many languages.
+
+You should prefer phonemizers trained on full sentences vs individual words if available
+
+The core phonemizer classes are summarized in the table below, listing the supported languages, the source library they
+wrap, and the output alphabets they can generate.
+
+***
+
+| Language(s)         | Phonemizer Class       | Source/Library                                                                                                     | Output Alphabets             | 
+|:--------------------|:-----------------------|:-------------------------------------------------------------------------------------------------------------------|:-----------------------------|
+| **Multilingual**    | `ByT5Phonemizer`       | [OpenVoiceOS ByT5](https://huggingface.co/collections/OpenVoiceOS/g2p-models-6886a8d612825c3fe65befa0) ONNX Models | IPA                          | High-quality, model-based G2P for an extensive list of languages.                                                                           |
+| **Multilingual**    | `CharsiuPhonemizer`    | [Charsiu](https://github.com/lingjzhu/CharsiuG2P) ByT5 ONNX Model                                                  | IPA                          | Very extensive multilingual support, including many regional dialects and variants (e.g., `eng-uk`, `spa-me`, `zho-s`).                     |
+| **Multilingual**    | `EspeakPhonemizer`     | `espeak-ng` command-line tool                                                                                      | IPA                          | Broad language coverage, relying on the widely-used `espeak-ng` engine.                                                                     |
+| **Multilingual**    | `GruutPhonemizer`      | [gruut](https://github.com/rhasspy/gruut)                                                                          | IPA                          | A tokenizer, text cleaner, and IPA phonemizer for several human languages that supports SSML.                                               |
+| **Multilingual**    | `MisakiPhonemizer`     | [misaki](https://github.com/hexgrad/misaki)                                                                        | IPA                          | Misaki is a G2P engine designed for Kokoro models.                                                                                          |
+| **Multilingual**    | `TransphonePhonemizer` | [transphone](https://github.com/xinjli/transphone)                                                                 | IPA                          | It provides approximated phoneme tokenizers and G2P model for 7546 languages registered in the Glottolog database.                          |
+| **Multilingual**    | `EpitranPhonemizer`    | [epitran](https://github.com/dmort27/epitran)                                                                      | IPA                          | A tool for transcribing orthographic text as IPA                                                                                            |
+| **Mirandese (mwl)** | `MirandesePhonemizer`  | [mwl_phonemizer](https://github.com/TigreGotico/mwl_phonemizer)                                                                 | IPA                          | A tool for transcribing orthographic text as IPA                                                                                            |
+| **Arabic (ar)**     | `MantoqPhonemizer`     | [mantoq](https://github.com/mush42/mantoq)                                                                         | BUCKWALTER, IPA              | Translates unvoweled Arabic to phonemes, with optional conversion to IPA.                                                                   |
+| **Chinese (zh)**    | `JiebaPhonemizer`      | [jieba](https://github.com/fxsjy/jieba)                                                                            | HANZI                        | Segments Chinese text into words with spaces; useful for pre-processing.                                                                    |
+| **Chinese (zh)**    | `G2pMPhonemizer`       | [g2pC](https://github.com/Kyubyong/g2pC)                                                                           | IPA, Pinyin                  | CRF-based Grapheme-to-Phoneme converter                                                                                                     |
+| **Chinese (zh)**    | `G2pMPhonemizer`       | [g2pm](https://github.com/kakaobrain/g2pm)                                                                         | IPA, Pinyin                  | A Neural Grapheme-to-Phoneme Conversion Package for Mandarin Chinese                                                                        |
+| **Chinese (zh)**    | `XpinyinPhonemizer`    | [xpinyin](https://github.com/lxneng/xpinyin)                                                                       | IPA, Pinyin                  | basic pinyin generator with optional tone marks                                                                                             |
+| **Chinese (zh)**    | `PypinyinPhonemizer`   | [pypinyin](https://github.com/rainforest32/pypinyin)                                                               | IPA, Pinyin                  | comprehensive and accurate pinyin library                                                                                                   |
+| **English (en)**    | `G2PEnPhonemizer`      | [g2pE](https://github.com/Kyubyong/g2p)                                                                            | IPA                          | A deep learning seq2seq framework based on TensorFlow                                                                                       |
+| **English (en)**    | `OpenPhonemizer`       | [OpenPhonemizer](https://github.com/NeuralVox/OpenPhonemizer)                                                      | IPA                          | IPA Phonemizer powered by deep learning. This Phonemizer attempts to replicate the espeak Phonemizer while remaining permissively-licensed. |
+| **English (en)**    | `DeepPhonemizer`       | [DeepPhonemizer](https://github.com/spring-media/DeepPhonemizer)                                                   | IPA / ARPA                   | Uses pre-trained deep learning models for English.                                                                                          |
+| **Galician (gl)**   | `CotoviaPhonemizer`    | [cotovia](https://github.com/TigreGotico/cotovia-mirror)                                                           | IPA, Native Cotovia Phonemes | Relies on the `cotovia`executable for Galician phonemization.                                                                               |
+| **Hebrew (he)**     | `PhonikudPhonemizer`   | [phonikud](https://github.com/thewh1teagle/phonikud)                                                               | IPA                          | Converts Hebrew text to IPA phonemes.                                                                                                       |
+| **Japanese (ja)**   | `OpenJTaklPhonemizer`  | [pyopenjtalk](https://github.com/r9y9/pyopenjtalk)                                                                 | HEPBURN, KANA                | High-quality Japanese G2P.                                                                                                                  |
+| **Japanese (ja)**   | `CutletPhonemizer`     | [cutlet](https://github.com/polm/cutlet)                                                                           | HEPBURN, KUNREI, NIHON       | Provides various Romanization standards.                                                                                                    |
+| **Japanese (ja)**   | `PyKakasiPhonemizer`   | [pykakasi](https://codeberg.org/miurahr/pykakasi)                                                                  | HEPBURN, KANA, HIRA          | Romanization and Kana conversion.                                                                                                           |
+| **Korean (ko)**     | `G2PKPhonemizer`       | [g2pK](https://github.com/Kyubyong/g2pK)                                                                           | IPA, HANGUL                  | Provides G2P for Korean, with optional IPA conversion.                                                                                      |
+| **Korean (ko)**     | `KoG2PPhonemizer`      | [KoG2P](https://github.com/scarletcho/KoG2P)                                                                       | IPA, HANGUL                  | Provides G2P for Korean, with optional IPA conversion.                                                                                      |
+| **Persian (fa)**    | `PersianPhonemizer`    | [persian_phonemizer](https://github.com/de-mh/persian_phonemizer)                                                  | ERAAB, IPA                   | Supports both standard IPA and the native ERAAB (diacritical) representations.                                                              |
+| **Vietnamese (vi)** | `VIPhonemePhonemizer`  | [Viphoneme](https://github.com/v-nhandt21/Viphoneme)                                                               | IPA                          | Uses the `viphoneme` library for Vietnamese G2P.                                                                                            |
+
+### Credits
+
+Phoonnx is built in the shoulders of giants
+
+- [jaywalnut310/vits](https://github.com/jaywalnut310/vits) - the original VITS implementation, the back-bone architecture of phoonnx models
+- [MycroftAI/mimic3](https://github.com/MycroftAI/mimic3) and [rhasspy/piper](https://github.com/rhasspy/piper) - for inspiration and reference implementation of a phonemizer for pre-processing inputs
+
+Individual languages greatly benefit from domain-specific knowledge, for convenience phoonnx also bundles code from
+
+- [uvigo/cotovia](https://github.com/TigreGotico/cotovia-mirror) for galician phonemization (pre-compiled binaries bundled)
+- [mush42/mantoq](https://github.com/mush42/mantoq) for arabic phonemization
+- [mush42/libtashkeel](https://github.com/mush42/libtashkeel) for arabic diacritics
+- [scarletcho/KoG2P](https://github.com/scarletcho/KoG2P) for korean phonemization
+- [stannam/hangul_to_ipa](https://github.com/stannam/hangul_to_ipa) a converter from Hangul to IPA
+- [chorusai/arpa2ipa](https://github.com/chorusai/arpa2ipa) a converter from Arpabet to IPA
+- [PaddlePaddle/PaddleSpeech](https://github.com/PaddlePaddle/PaddleSpeech/blob/8097a56be811a540f4f62a95a9094296c374351a/paddlespeech/t2s/frontend/zh_normalization/) for chinese number verbalization
+

{phoonnx-0.1.0a1 → phoonnx-0.1.1a1}/phoonnx/config.py

@@ -33,7 +33,7 @@ class Alphabet(str, Enum):
     ERAAB = "eraab" # fa
     COTOVIA = "cotovia" # gl
     HANZI = "hanzi" # zh
-    MANTOQ = "mantoq" # ar
+    BUCKWALTER = "buckwalter" # ar
 
 
 
@@ -48,6 +48,8 @@ class PhonemeType(str, Enum):
     EPITRAN = "epitran"
     BYT5 = "byt5"
     CHARSIU = "charsiu"  # technically same as byt5, but needs special handling for whitespace
+    TRANSPHONE = "transphone"
+    MIRANDESE = "mwl_phonemizer"
 
     DEEPPHONEMIZER = "deepphonemizer" # en
     OPENPHONEMIZER = "openphonemizer" # en
@@ -382,6 +384,9 @@ class SynthesisConfig:
 
     enable_phonetic_spellings: bool = True
 
+    """for arabic and hebrew models"""
+    add_diacritics: bool = True
+
 
 def get_phonemizer(phoneme_type: PhonemeType,
                    alphabet: Alphabet = Alphabet.IPA,
@@ -389,6 +394,7 @@ def get_phonemizer(phoneme_type: PhonemeType,
     from phoonnx.phonemizers import (EpitranPhonemizer, EspeakPhonemizer, OpenPhonemizer, OpenJTaklPhonemizer,
                                      ByT5Phonemizer, CharsiuPhonemizer, DeepPhonemizer, PersianPhonemizer,
                                      G2pCPhonemizer, G2pMPhonemizer, G2PKPhonemizer, G2PEnPhonemizer,
+                                     TransphonePhonemizer, MirandesePhonemizer,
                                      GruutPhonemizer, GraphemePhonemizer, MantoqPhonemizer, MisakiPhonemizer,
                                      KoG2PPhonemizer, PypinyinPhonemizer, PyKakasiPhonemizer, CotoviaPhonemizer,
                                      CutletPhonemizer, PhonikudPhonemizer, VIPhonemePhonemizer, XpinyinPhonemizer,
@@ -405,6 +411,10 @@ def get_phonemizer(phoneme_type: PhonemeType,
         phonemizer = EpitranPhonemizer()
     elif phoneme_type == PhonemeType.MISAKI:
         phonemizer = MisakiPhonemizer()
+    elif phoneme_type == PhonemeType.TRANSPHONE:
+        phonemizer = TransphonePhonemizer()
+    elif phoneme_type == PhonemeType.MIRANDESE:
+        phonemizer = MirandesePhonemizer()
     elif phoneme_type == PhonemeType.DEEPPHONEMIZER:
         phonemizer = DeepPhonemizer(model)
     elif phoneme_type == PhonemeType.OPENPHONEMIZER:

{phoonnx-0.1.0a1 → phoonnx-0.1.1a1}/phoonnx/phonemizers/__init__.py

@@ -12,7 +12,8 @@ from phoonnx.phonemizers.ko import KoG2PPhonemizer, G2PKPhonemizer
 from phoonnx.phonemizers.zh import (G2pCPhonemizer, G2pMPhonemizer, PypinyinPhonemizer,
                                     XpinyinPhonemizer, JiebaPhonemizer)
 from phoonnx.phonemizers.mul import (EspeakPhonemizer, EpitranPhonemizer, MisakiPhonemizer,
-                                     GruutPhonemizer, ByT5Phonemizer, CharsiuPhonemizer)
+                                     GruutPhonemizer, ByT5Phonemizer, CharsiuPhonemizer, TransphonePhonemizer)
+from phoonnx.phonemizers.mwl import MirandesePhonemizer
 
 Phonemizer = Union[
     MisakiPhonemizer,
@@ -22,6 +23,8 @@ Phonemizer = Union[
     EspeakPhonemizer,
     GruutPhonemizer,
     EpitranPhonemizer,
+    TransphonePhonemizer,
+    MirandesePhonemizer,
     OpenJTaklPhonemizer,
     CutletPhonemizer,
     PyKakasiPhonemizer,

phoonnx-0.1.1a1/phoonnx/phonemizers/ar.py

@@ -0,0 +1,98 @@
+from phoonnx.config import Alphabet
+from phoonnx.phonemizers.base import BasePhonemizer
+from phoonnx.thirdparty.bw2ipa import translate as bw2ipa
+from phoonnx.thirdparty.mantoq import g2p as mantoq
+
+
+class MantoqPhonemizer(BasePhonemizer):
+
+    def __init__(self, alphabet=Alphabet.BUCKWALTER):
+        if alphabet not in [Alphabet.IPA, Alphabet.BUCKWALTER]:
+            raise ValueError("unsupported alphabet")
+        super().__init__(alphabet)
+
+    @classmethod
+    def get_lang(cls, target_lang: str) -> str:
+        """
+        Validates and returns the closest supported language code.
+
+        Args:
+            target_lang (str): The language code to validate.
+
+        Returns:
+            str: The validated language code.
+
+        Raises:
+            ValueError: If the language code is unsupported.
+        """
+        # this check is here only to throw an exception if invalid language is provided
+        return cls.match_lang(target_lang, ["ar"])
+
+    def phonemize_string(self, text: str, lang: str = "ar") -> str:
+        """
+        Phonemizes an Arabic string using the Mantoq G2P tool.
+        If the alphabet is set to IPA, it then converts the result using bw2ipa.
+        """
+        lang = self.get_lang(lang)
+        # The mantoq function returns a tuple of (normalized_text, phonemes)
+        normalized_text, phonemes = mantoq(text)
+
+        # The phonemes are a list of characters, we join them into a string
+        # and replace the word separator token with a space.
+        phonemes = "".join(phonemes).replace("_+_", " ")
+
+        if self.alphabet == Alphabet.IPA:
+            # If the alphabet is IPA, we use the bw2ipa function to translate
+            # the Buckwalter-like phonemes into IPA.
+            return bw2ipa(phonemes)
+
+        # Otherwise, we return the phonemes in the default Mantoq alphabet.
+        return phonemes
+
+
+if __name__ == "__main__":
+    from phoonnx.phonemizers.mul import EspeakPhonemizer
+
+    espeak = EspeakPhonemizer()
+
+    # Initialize phonemizers for both MANTOQ and IPA alphabets
+    pho_mantoq = MantoqPhonemizer(alphabet=Alphabet.IPA)
+
+
+    def compare(text):
+        print(f"Original Text: {text}")
+        print(f"  Mantoq: {pho_mantoq.phonemize_string(text, 'ar')}")
+        print(f"  Espeak: {espeak.phonemize_string(text, 'ar')}")
+
+        ts = pho_mantoq.add_diacritics(text, 'ar')
+        print(f"Tashkeel Text: {ts}")
+        print(f"  Mantoq: {pho_mantoq.phonemize_string(ts, 'ar')}")
+        print(f"  Espeak: {espeak.phonemize_string(ts, 'ar')}")
+        print("\n#########################")
+
+
+    text = "مرحبا بالعالم"
+    compare(text)
+
+    text = "ذهب الطالب إلى المكتبة لقراءة كتاب عن تاريخ الأندلس."
+    compare(text)
+
+    # 1. Test for gemination of a sun letter (e.g., ash-shams)
+    text = "الشمس"
+    compare(text)
+
+    # 2. Test for long vowels (e.g., 'fil' - elephant)
+    text = "فيل"
+    compare(text)
+
+    # 3. Test for glide (e.g., 'yawm' - day)
+    text = "يوم"
+    compare(text)
+
+    # 4. Test for long vowels (e.g., 'suwr' - wall)
+    text = "سور"
+    compare(text)
+
+    # 5. Test for glide (e.g., 'law' - if)
+    text = "لو"
+    compare(text)

{phoonnx-0.1.0a1 → phoonnx-0.1.1a1}/phoonnx/phonemizers/base.py

@@ -8,6 +8,8 @@ from langcodes import tag_distance
 from quebra_frases import sentence_tokenize
 from phoonnx.config import Alphabet
 from phoonnx.util import normalize
+from phoonnx.thirdparty.phonikud import PhonikudDiacritizer
+from phoonnx.thirdparty.tashkeel import TashkeelDiacritizer
 
 # list of (substring, terminator, end_of_sentence) tuples.
 TextChunks = List[Tuple[str, str, bool]]
@@ -18,10 +20,27 @@ PhonemizedChunks = list[list[str]]
 
 
 class BasePhonemizer(metaclass=abc.ABCMeta):
-    def __init__(self, alphabet: Alphabet = Alphabet.UNICODE):
+    def __init__(self, alphabet: Alphabet = Alphabet.UNICODE,
+                 taskeen_threshold: Optional[float] = 0.8):
         super().__init__()
         self.alphabet = alphabet
 
+        self.taskeen_threshold = taskeen_threshold  # arabic only
+        self._tashkeel: Optional[TashkeelDiacritizer] = None
+        self._phonikud: Optional[PhonikudDiacritizer] = None # hebrew only
+
+    @property
+    def phonikud(self) -> PhonikudDiacritizer:
+        if self._phonikud is None:
+            self._phonikud = PhonikudDiacritizer()
+        return self._phonikud
+
+    @property
+    def tashkeel(self) -> TashkeelDiacritizer:
+        if self._tashkeel is None:
+            self._tashkeel = TashkeelDiacritizer()
+        return self._tashkeel
+
     @abc.abstractmethod
     def phonemize_string(self, text: str, lang: str) -> str:
         raise NotImplementedError
@@ -29,6 +48,13 @@ class BasePhonemizer(metaclass=abc.ABCMeta):
     def phonemize_to_list(self, text: str, lang: str) -> List[str]:
         return list(self.phonemize_string(text, lang))
 
+    def add_diacritics(self, text: str, lang: str) -> str:
+        if lang.startswith("he"):
+            return self.phonikud.diacritize(text)
+        elif lang.startswith("ar"):
+            return self.tashkeel.diacritize(text, self.taskeen_threshold)
+        return text
+
     def phonemize(self, text: str, lang: str) -> PhonemizedChunks:
         if not text:
             return [('', '', True)]

{phoonnx-0.1.0a1 → phoonnx-0.1.1a1}/phoonnx/phonemizers/he.py

@@ -1,30 +1,12 @@
-import os.path
-
-import requests
-
-from phoonnx.phonemizers.base import BasePhonemizer
 from phoonnx.config import Alphabet
+from phoonnx.phonemizers.base import BasePhonemizer
 
 
 class PhonikudPhonemizer(BasePhonemizer):
-    dl_url = "https://huggingface.co/thewh1teagle/phonikud-onnx/resolve/main/phonikud-1.0.int8.onnx"
 
-    def __init__(self, model: str = None, diacritics=True):
-        from phonikud_onnx import Phonikud
+    def __init__(self):
         from phonikud import phonemize
         self.g2p = phonemize
-        self.diacritics = diacritics
-        if model is None:
-            base_path = os.path.expanduser("~/.local/share/phonikud")
-            fname = self.dl_url.split("/")[-1]
-            model = f"{base_path}/{fname}"
-            if not os.path.isfile(model):
-                os.makedirs(base_path, exist_ok=True)
-                # TODO - streaming download
-                data = requests.get(self.dl_url).content
-                with open(model, "wb") as f:
-                    f.write(data)
-        self.phonikud = Phonikud(model) if diacritics else None
         super().__init__(Alphabet.IPA)
 
     @classmethod
@@ -48,20 +30,19 @@ class PhonikudPhonemizer(BasePhonemizer):
         """
         """
         lang = self.get_lang(lang)
-        if self.diacritics:
-            text = self.phonikud.add_diacritics(text)
         return self.g2p(text)
 
 
 if __name__ == "__main__":
-    #text = "מתכת יקרה"
+    # text = "מתכת יקרה"
     text = 'שָׁלוֹם עוֹלָם'
 
-    pho = PhonikudPhonemizer(diacritics=False)
+    pho = PhonikudPhonemizer()
     lang = "he"
 
     print(f"\n--- Getting phonemes for '{text}' ---")
+    # text = pho.add_diacritics(text, lang)
     phonemes = pho.phonemize(text, lang)
     print(f"  Phonemes: {phonemes}")
     # --- Getting phonemes for 'שָׁלוֹם עוֹלָם' ---
-    #   Phonemes: [('ʃalˈom ʔolˈam', '.', True)]
+    #   Phonemes: [('ʃalˈom ʔolˈam', '.', True)]