phoonnx 0.0.0__py3-none-any.whl

phoonnx/thirdparty/mantoq/buck/tokenization.py

@@ -0,0 +1,105 @@
+from .phonetise_buckwalter import (arabic_to_buckwalter, buckwalter_to_arabic,
+                                   process_utterance)
+from .symbols import DOUBLING_TOKEN, EOS_TOKEN, SEPARATOR_TOKEN, symbols
+
+vowels = [
+    "aa",
+    "AA",
+    "uu0",
+    "uu1",
+    "UU0",
+    "UU1",
+    "ii0",
+    "ii1",
+    "II0",
+    "II1",
+    "a",
+    "A",
+    "u0",
+    "u1",
+    "U0",
+    "U1",
+    "i0",
+    "i1",
+    "I0",
+    "I1",
+]
+
+vowel_map = {
+    "aa": "aa",
+    "AA": "aa",
+    "uu0": "uu",
+    "uu1": "uu",
+    "UU0": "uu",
+    "UU1": "uu",
+    "ii0": "ii",
+    "ii1": "ii",
+    "II0": "ii",
+    "II1": "ii",
+    "a": "a",
+    "A": "a",
+    "u0": "u",
+    "u1": "u",
+    "U0": "u",
+    "U1": "u",
+    "i0": "i",
+    "i1": "i",
+    "I0": "i",
+    "I1": "i",
+}
+
+phon_to_id_ = {phon: i for i, phon in enumerate(symbols)}
+
+
+def tokens_to_ids(phonemes, phon_to_id=None):
+    if phon_to_id is None:
+        return [phon_to_id_[phon] for phon in phonemes]
+    return [phon_to_id[phon] for phon in phonemes]
+
+
+def ids_to_tokens(ids):
+    return [symbols[id] for id in ids]
+
+
+def arabic_to_phonemes(arabic):
+    buckw = arabic_to_buckwalter(arabic)
+    return process_utterance(buckw)
+
+
+def buckwalter_to_phonemes(buckw):
+    return process_utterance(buckw)
+
+
+def phonemes_to_tokens(phonemes: str, append_space=False):
+    phonemes = phonemes.replace("sil", "").replace("+", "_+_").split()
+    for i, phon in enumerate(phonemes):
+        if len(phon) == 2 and phon not in vowels and phon[0] == phon[1]:
+            phonemes[i] = phon[0]
+            phonemes.insert(i + 1, DOUBLING_TOKEN)
+        if phonemes[i] in vowels:
+            phonemes[i] = vowel_map[phonemes[i]]
+
+    if append_space:
+        phonemes.append(SEPARATOR_TOKEN)
+
+    phonemes.append(EOS_TOKEN)
+
+    return phonemes
+
+
+def buckwalter_to_tokens(buckw, append_space=False):
+    phonemes = buckwalter_to_phonemes(buckw)
+    tokens = phonemes_to_tokens(phonemes, append_space=append_space)
+    return tokens
+
+
+def arabic_to_tokens(arabic, append_space=False):
+    buckw = arabic_to_buckwalter(arabic)
+    tokens = buckwalter_to_tokens(buckw, append_space=append_space)
+    return tokens
+
+
+def simplify_phonemes(phonemes):
+    for k, v in vowel_map.items():
+        phonemes = phonemes.replace(k, v)
+    return phonemes

phoonnx/thirdparty/mantoq/num2words.py

@@ -0,0 +1,37 @@
+import re
+from functools import partial
+
+from phoonnx.thirdparty.mantoq.pyarabic import araby
+from phoonnx.thirdparty.mantoq.pyarabic import number as arnum
+from phoonnx.thirdparty.mantoq.pyarabic.trans import normalize_digits
+
+NUM_REGEX = re.compile(r"\d+")
+PERCENT_NO_DIAC = "بالمئة"
+PERCENT_DIAC = "بِالْمِئَة"
+
+
+def _convert_num2words(m: re.Match, *, apply_tashkeel):
+    number = m.group(0)
+    word_representation = arnum.number2text(number)
+    if apply_tashkeel:
+        return " ".join(arnum.pre_tashkeel_number(word_representation.split(" ")))
+    return word_representation
+
+
+def num2words(text: str, handle_percent=True, apply_tashkeel: bool = True) -> str:
+    """
+    Converts numbers in `text` to Arabic words.
+    Simple conversion. Does not check if the number is date/currency...etc.
+
+    Args:
+        text: input text that may contain numbers
+        apply_tashkeel: diacritize added words
+    """
+    text = normalize_digits(text)
+    output = NUM_REGEX.sub(
+        partial(_convert_num2words, apply_tashkeel=apply_tashkeel), text
+    )
+    if handle_percent:
+        replacement = PERCENT_DIAC if apply_tashkeel else PERCENT_NO_DIAC
+        output = output.replace("%", f" {replacement}")
+    return araby.fix_spaces(output)

phoonnx/thirdparty/mantoq/pyarabic/__init__.py

@@ -0,0 +1,12 @@
+# PyArabic -- Regression testing
+#
+# Copyright (C) 2010 Taha Zerrouki
+# Author: Taha Zerrouki <taha zerrouki at gawab dot com>
+# URL: <http://pyarabic.sf.net>
+#
+# $Id: __init__.py :
+
+"""
+Main.
+"""
+__docformat__ = "epytext en"

phoonnx/thirdparty/mantoq/pyarabic/arabrepr.py

@@ -0,0 +1,64 @@
+#!/usr/bin/python
+# -*- coding: UTF-8 -*-
+"""
+Improve repr predifined function to best display of objects containing unicode
+Unicode represention texts
+@author: Taha Zerrouki
+@contact: taha dot zerrouki at gmail dot com
+@copyright: Taha Zerrouki
+@license: GPL
+@date:2014/03/01
+@version: 0.1
+"""
+from __future__ import (absolute_import, division, print_function,
+                        unicode_literals)
+
+import sys
+
+if sys.version_info < (3, 0):
+    import repr as reprlib
+else:
+    import reprlib
+
+
+class ArabicRepr(reprlib.Repr):
+    """A redifinition of repr fucntion,
+    you can use it like this
+
+    Example:
+        >>> import pyarabic.arabrepr as arabrepr
+        >>> arepr = arabrepr.ArabicRepr()
+        >>> repr = arepr.repr
+        >>> word = u"السلام عليكم ورحمة الله"
+        >>> wordlist = word.split(" ")
+        >>> print wordlist
+        [u'\u0627\u0644\u0633\u0644\u0627\u0645',
+        u'\u0639\u0644\u064a\u0643\u0645',
+        u'\u0648\u0631\u062d\u0645\u0629',
+        u'\u0627\u0644\u0644\u0647']
+        >>> print repr(wordlist)
+        [u'السلام', u'عليكم', u'ورحمة', u'الله']
+    """
+
+    def repr_unicode(self, obj, level):
+        "Modify unicode display"
+        return "u'%s'" % obj
+
+    def arepr_unicode(self, obj, level):
+        "Modify unicode display"
+        return "u'%s'" % obj
+
+
+if sys.version_info < (3, 0):
+
+    def arepr(data):
+        """display a dict with arabic text properly"""
+        return repr(data).replace("},", "},\n").decode("unicode-escape").encode("utf8")
+
+else:
+
+    def arepr(data):
+        """display a dict with arabic text properly"""
+        return repr(data).replace(
+            "},", "},\n"
+        )  # .decode('unicode-escape').encode('utf8')