phoonnx 0.0.0__py3-none-any.whl

phoonnx/thirdparty/tashkeel/LICENSE

@@ -0,0 +1,22 @@
+MIT License
+
+Copyright (c) 2023 Musharraf
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.
+

phoonnx/thirdparty/tashkeel/SOURCE

@@ -0,0 +1 @@
+https://github.com/mush42/libtashkeel

phoonnx/thirdparty/tashkeel/__init__.py

@@ -0,0 +1,212 @@
+"""Python implementation of libtashkeel.
+
+See: https://github.com/mush42/libtashkeel
+
+Ported with the help of ChatGPT 2025-05-01.
+"""
+
+import json
+from pathlib import Path
+from typing import Optional, Union
+
+import numpy as np
+from onnxruntime import InferenceSession
+
+TASHKEEL_DIR = Path(__file__).parent
+CHAR_LIMIT = 12000
+PAD = "_"
+NUMERAL_SYMBOL = "#"
+NUMERALS = set("0123456789٠١٢٣٤٥٦٧٨٩")
+HARAKAT_CHARS = {"\u064c", "\u064d", "\u064e", "\u064f", "\u0650", "\u0651", "\u0652"}
+ARABIC_DIACRITICS = set(map(chr, [1618, 1617, 1614, 1615, 1616, 1611, 1612, 1613]))
+NORMALIZED_DIAC_MAP = {"َّ": "َّ", "ًّ": "ًّ", "ُّ": "ُّ", "ٌّ": "ٌّ", "ِّ": "ِّ", "ٍّ": "ٍّ"}
+SUKOON = chr(0x652)
+
+
+class TashkeelError(Exception):
+    """Error for tashkeel."""
+
+
+class TashkeelDiacritizer:
+    """Add diacritics for Arabic text with libtashkeel."""
+
+    def __init__(self, model_dir: Union[str, Path] = TASHKEEL_DIR) -> None:
+        """Initialize diacritizer."""
+        model_dir = Path(model_dir)
+        self.session = InferenceSession(model_dir / "model.onnx")
+
+        # Load JSON maps
+        with open(
+            model_dir / "input_id_map.json", "r", encoding="utf-8"
+        ) as input_id_map_file:
+            self.input_id_map: dict[str, int] = json.load(input_id_map_file)
+
+        with open(
+            model_dir / "target_id_map.json", "r", encoding="utf-8"
+        ) as target_id_map_file:
+            target_id_map: dict[str, int] = json.load(target_id_map_file)
+            self.id_target_map: dict[int, str] = {
+                i: c for c, i in target_id_map.items()
+            }
+
+        self.target_id_meta_chars: set[int] = {target_id_map[c] for c in [PAD]}
+
+        with open(
+            model_dir / "hint_id_map.json", "r", encoding="utf-8"
+        ) as hint_id_map_file:
+            self.hint_id_map: dict[str, int] = json.load(hint_id_map_file)
+
+    def __call__(self, text: str, taskeen_threshold: Optional[float] = None) -> str:
+        """Add diacritics using libtashkeel."""
+        return self.diacritize(text)
+
+    def diacritize(self, text: str, taskeen_threshold=None) -> str:
+        """Add diacritics using libtashkeel."""
+        text = text.strip()
+
+        if len(text) > CHAR_LIMIT:
+            raise TashkeelError(f"Text length cannot exceed {CHAR_LIMIT}")
+
+        input_text, removed_chars = self._to_valid_chars(text)
+        input_text, diacritics = self._extract_chars_and_diacritics(
+            input_text, normalize_diacritics=True
+        )
+
+        input_ids = self._input_to_ids(input_text)
+        diac_ids = self._hint_to_ids(diacritics)
+        seq_length = len(input_ids)
+
+        if seq_length == 0:
+            return text
+
+        target_ids, logits = self._infer(input_ids, diac_ids, seq_length)
+
+        diacritics = self._target_to_diacritics(target_ids)
+        if taskeen_threshold is None:
+            return self._annotate_text_with_diacritics(text, diacritics, removed_chars)
+
+        return self._annotate_text_with_diacritics_taskeen(
+            text, diacritics, removed_chars, logits, taskeen_threshold
+        )
+
+    def _infer(
+        self, input_ids: list[int], diac_ids: list[int], seq_length: int
+    ) -> tuple[list[int], list[float]]:
+        """Infer target ids and logits."""
+        input_ids_arr = np.array(input_ids, dtype=np.int64).reshape(1, seq_length)
+        diac_ids_arr = np.array(diac_ids, dtype=np.int64).reshape(1, seq_length)
+        input_len_arr = np.array([seq_length], dtype=np.int64).reshape(1)
+
+        inputs = {
+            "char_inputs": input_ids_arr,
+            "diac_inputs": diac_ids_arr,
+            "input_lengths": input_len_arr,
+        }
+
+        outputs = self.session.run(None, inputs)
+
+        # Output 0: target_ids (u8)
+        # Output 1: logits (f32)
+        target_ids = outputs[0].flatten().astype(np.uint8).tolist()
+        logits = outputs[1].flatten().astype(np.float32).tolist()
+
+        return target_ids, logits
+
+    def _annotate_text_with_diacritics(
+        self, input_text: str, diacritics: list[str], removed_chars: set[str]
+    ) -> str:
+        output: list[str] = []
+        diac_iter = iter(diacritics)
+        for c in input_text:
+            if self._is_diacritic_char(c):
+                continue
+
+            if c in removed_chars:
+                output.append(c)
+            else:
+                output.append(c)
+                output.append(next(diac_iter, ""))
+
+        return "".join(output)
+
+    def _annotate_text_with_diacritics_taskeen(
+        self,
+        input_text: str,
+        diacritics: list[str],
+        removed_chars: set[str],
+        logits: list[float],
+        threshold: float,
+    ) -> str:
+        output: list[str] = []
+        diac_iter = zip(diacritics, logits)
+        for c in input_text:
+            if self._is_diacritic_char(c):
+                continue
+
+            if c in removed_chars:
+                output.append(c)
+            else:
+                output.append(c)
+                diac, logit = next(diac_iter, ("", 0.0))
+                if logit > threshold:
+                    output.append(SUKOON)
+                else:
+                    output.append(diac)
+        return "".join(output)
+
+    def _is_diacritic_char(self, c) -> bool:
+        return c in ARABIC_DIACRITICS
+
+    def _extract_chars_and_diacritics(
+        self, text: str, normalize_diacritics: bool = True
+    ) -> tuple[str, list[str]]:
+        text = text.lstrip("".join(ARABIC_DIACRITICS))
+
+        clean_chars = []
+        diacritics = []
+        pending_diac = ""
+
+        for c in list(text) + [" "]:  # emulate .chain(iter::once(' '))
+            if self._is_diacritic_char(c):
+                pending_diac += c
+            else:
+                clean_chars.append(c)
+                diacritics.append(pending_diac)
+                pending_diac = ""
+
+        if clean_chars:
+            clean_chars.pop()  # pop the trailing space equivalent
+        if diacritics:
+            diacritics.pop(0)  # remove initial empty
+
+        if normalize_diacritics:
+            for i, d in enumerate(diacritics):
+                if d not in self.hint_id_map:
+                    diacritics[i] = NORMALIZED_DIAC_MAP.get(d, "")
+
+        return "".join(clean_chars), diacritics
+
+    def _to_valid_chars(self, text: str) -> tuple[str, set[str]]:
+        valid: list[str] = []
+        invalid: set[str] = set()
+        for c in text:
+            if (c in self.input_id_map) or (c in ARABIC_DIACRITICS):
+                valid.append(c)
+            elif c in NUMERALS:
+                valid.append(NUMERAL_SYMBOL)
+            else:
+                invalid.add(c)
+        return "".join(valid), invalid
+
+    def _input_to_ids(self, text: str) -> list[int]:
+        return [self.input_id_map[c] for c in text]
+
+    def _hint_to_ids(self, diacritics: list[str]) -> list[int]:
+        return [self.hint_id_map[d] for d in diacritics]
+
+    def _target_to_diacritics(self, target_ids: list[int]) -> list[str]:
+        return [
+            self.id_target_map[i]
+            for i in target_ids
+            if i not in self.target_id_meta_chars
+        ]

phoonnx/thirdparty/tashkeel/hint_id_map.json

@@ -0,0 +1,18 @@
+{
+  "_": 0,
+  "": 1,
+  "ً": 2,
+  "ٌ": 3,
+  "ٍ": 4,
+  "َ": 5,
+  "ُ": 6,
+  "ِ": 7,
+  "ًّ": 8,
+  "ٌّ": 9,
+  "ٍّ": 10,
+  "َّ": 11,
+  "ُّ": 12,
+  "ِّ": 13,
+  "ْ": 14,
+  "ّ": 15
+}

phoonnx/thirdparty/tashkeel/input_id_map.json

@@ -0,0 +1,56 @@
+{
+  "_": 0,
+  " ": 1,
+  "#": 2,
+  "!": 3,
+  "\"": 4,
+  "(": 5,
+  ")": 6,
+  "-": 7,
+  ".": 8,
+  "/": 9,
+  ":": 10,
+  "[": 11,
+  "]": 12,
+  "«": 13,
+  "»": 14,
+  "،": 15,
+  "؛": 16,
+  "؟": 17,
+  "ء": 18,
+  "آ": 19,
+  "أ": 20,
+  "ؤ": 21,
+  "إ": 22,
+  "ئ": 23,
+  "ا": 24,
+  "ب": 25,
+  "ة": 26,
+  "ت": 27,
+  "ث": 28,
+  "ج": 29,
+  "ح": 30,
+  "خ": 31,
+  "د": 32,
+  "ذ": 33,
+  "ر": 34,
+  "ز": 35,
+  "س": 36,
+  "ش": 37,
+  "ص": 38,
+  "ض": 39,
+  "ط": 40,
+  "ظ": 41,
+  "ع": 42,
+  "غ": 43,
+  "ف": 44,
+  "ق": 45,
+  "ك": 46,
+  "ل": 47,
+  "م": 48,
+  "ن": 49,
+  "ه": 50,
+  "و": 51,
+  "ى": 52,
+  "ي": 53
+}

phoonnx/thirdparty/tashkeel/target_id_map.json

@@ -0,0 +1,17 @@
+{
+  "_": 0,
+  "": 1,
+  "ً": 2,
+  "ٌ": 3,
+  "ٍ": 4,
+  "َ": 5,
+  "ُ": 6,
+  "ِ": 7,
+  "ًّ": 8,
+  "ٌّ": 9,
+  "ٍّ": 10,
+  "َّ": 11,
+  "ُّ": 12,
+  "ِّ": 13,
+  "ْ": 14
+}

phoonnx/thirdparty/zh_num.py

@@ -0,0 +1,238 @@
+# Copyright (c) 2021 PaddlePaddle Authors. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+"""
+Rules to verbalize numbers into Chinese characters.
+https://zh.wikipedia.org/wiki/中文数字#現代中文
+"""
+import re
+from collections import OrderedDict
+from typing import List
+
+DIGITS = {str(i): tran for i, tran in enumerate('零一二三四五六七八九')}
+UNITS = OrderedDict({
+    1: '十',
+    2: '百',
+    3: '千',
+    4: '万',
+    8: '亿',
+})
+
+COM_QUANTIFIERS = '(封|艘|把|目|套|段|人|所|朵|匹|张|座|回|场|尾|条|个|首|阙|阵|网|炮|顶|丘|棵|只|支|袭|辆|挑|担|颗|壳|窠|曲|墙|群|腔|砣|座|客|贯|扎|捆|刀|令|打|手|罗|坡|山|岭|江|溪|钟|队|单|双|对|出|口|头|脚|板|跳|枝|件|贴|针|线|管|名|位|身|堂|课|本|页|家|户|层|丝|毫|厘|分|钱|两|斤|担|铢|石|钧|锱|忽|(千|毫|微)克|毫|厘|(公)分|分|寸|尺|丈|里|寻|常|铺|程|(千|分|厘|毫|微)米|米|撮|勺|合|升|斗|石|盘|碗|碟|叠|桶|笼|盆|盒|杯|钟|斛|锅|簋|篮|盘|桶|罐|瓶|壶|卮|盏|箩|箱|煲|啖|袋|钵|年|月|日|季|刻|时|周|天|秒|分|小时|旬|纪|岁|世|更|夜|春|夏|秋|冬|代|伏|辈|丸|泡|粒|颗|幢|堆|条|根|支|道|面|片|张|颗|块|元|(亿|千万|百万|万|千|百)|(亿|千万|百万|万|千|百|美|)元|(亿|千万|百万|万|千|百|十|)吨|(亿|千万|百万|万|千|百|)块|角|毛|分|(公(里|引|丈|尺|寸|分|釐)))'
+
+# 分数表达式
+RE_FRAC = re.compile(r'(-?)(\d+)/(\d+)')
+
+
+def replace_frac(match) -> str:
+    """
+    Args:
+        match (re.Match)
+    Returns:
+        str
+    """
+    sign = match.group(1)
+    nominator = match.group(2)
+    denominator = match.group(3)
+    sign: str = "负" if sign else ""
+    nominator: str = num2str(nominator)
+    denominator: str = num2str(denominator)
+    result = f"{sign}{denominator}分之{nominator}"
+    return result
+
+
+# 百分数表达式
+RE_PERCENTAGE = re.compile(r'(-?)(\d+(\.\d+)?)%')
+
+
+def replace_percentage(match) -> str:
+    """
+    Args:
+        match (re.Match)
+    Returns:
+        str
+    """
+    sign = match.group(1)
+    percent = match.group(2)
+    sign: str = "负" if sign else ""
+    percent: str = num2str(percent)
+    result = f"{sign}百分之{percent}"
+    return result
+
+
+# 整数表达式
+# 带负号的整数 -10
+RE_INTEGER = re.compile(r'(-)' r'(\d+)')
+
+
+def replace_negative_num(match) -> str:
+    """
+    Args:
+        match (re.Match)
+    Returns:
+        str
+    """
+    sign = match.group(1)
+    number = match.group(2)
+    sign: str = "负" if sign else ""
+    number: str = num2str(number)
+    result = f"{sign}{number}"
+    return result
+
+
+# 编号-无符号整形
+# 00078
+RE_DEFAULT_NUM = re.compile(r'\d{3}\d*')
+
+
+def replace_default_num(match):
+    """
+    Args:
+        match (re.Match)
+    Returns:
+        str
+    """
+    number = match.group(0)
+    return verbalize_digit(number, alt_one=True)
+
+
+# 数字表达式
+# 纯小数
+RE_DECIMAL_NUM = re.compile(r'(-?)((\d+)(\.\d+))' r'|(\.(\d+))')
+# 正整数 + 量词
+RE_POSITIVE_QUANTIFIERS = re.compile(r"(\d+)([多余几\+])?" + COM_QUANTIFIERS)
+RE_NUMBER = re.compile(r'(-?)((\d+)(\.\d+)?)' r'|(\.(\d+))')
+
+
+def replace_positive_quantifier(match) -> str:
+    """
+    Args:
+        match (re.Match)
+    Returns:
+        str
+    """
+    number = match.group(1)
+    match_2 = match.group(2)
+    if match_2 == "+":
+        match_2 = "多"
+    match_2: str = match_2 if match_2 else ""
+    quantifiers: str = match.group(3)
+    number: str = num2str(number)
+    result = f"{number}{match_2}{quantifiers}"
+    return result
+
+
+def replace_number(match) -> str:
+    """
+    Args:
+        match (re.Match)
+    Returns:
+        str
+    """
+    sign = match.group(1)
+    number = match.group(2)
+    pure_decimal = match.group(5)
+    if pure_decimal:
+        result = num2str(pure_decimal)
+    else:
+        sign: str = "负" if sign else ""
+        number: str = num2str(number)
+        result = f"{sign}{number}"
+    return result
+
+
+# 范围表达式
+# match.group(1) and match.group(8) are copy from RE_NUMBER
+
+RE_RANGE = re.compile(
+    r'((-?)((\d+)(\.\d+)?)|(\.(\d+)))[-~]((-?)((\d+)(\.\d+)?)|(\.(\d+)))')
+
+
+def replace_range(match) -> str:
+    """
+    Args:
+        match (re.Match)
+    Returns:
+        str
+    """
+    first, second = match.group(1), match.group(8)
+    first = RE_NUMBER.sub(replace_number, first)
+    second = RE_NUMBER.sub(replace_number, second)
+    result = f"{first}到{second}"
+    return result
+
+
+def _get_value(value_string: str, use_zero: bool=True) -> List[str]:
+    stripped = value_string.lstrip('0')
+    if len(stripped) == 0:
+        return []
+    elif len(stripped) == 1:
+        if use_zero and len(stripped) < len(value_string):
+            return [DIGITS['0'], DIGITS[stripped]]
+        else:
+            return [DIGITS[stripped]]
+    else:
+        largest_unit = next(
+            power for power in reversed(UNITS.keys()) if power < len(stripped))
+        first_part = value_string[:-largest_unit]
+        second_part = value_string[-largest_unit:]
+        return _get_value(first_part) + [UNITS[largest_unit]] + _get_value(
+            second_part)
+
+
+def verbalize_cardinal(value_string: str) -> str:
+    if not value_string:
+        return ''
+
+    # 000 -> '零' , 0 -> '零'
+    value_string = value_string.lstrip('0')
+    if len(value_string) == 0:
+        return DIGITS['0']
+
+    result_symbols = _get_value(value_string)
+    # verbalized number starting with '一十*' is abbreviated as `十*`
+    if len(result_symbols) >= 2 and result_symbols[0] == DIGITS[
+            '1'] and result_symbols[1] == UNITS[1]:
+        result_symbols = result_symbols[1:]
+    return ''.join(result_symbols)
+
+
+def verbalize_digit(value_string: str, alt_one=False) -> str:
+    result_symbols = [DIGITS[digit] for digit in value_string]
+    result = ''.join(result_symbols)
+    if alt_one:
+        result = result.replace("一", "幺")
+    return result
+
+
+def num2str(value_string: str) -> str:
+    integer_decimal = value_string.split('.')
+    if len(integer_decimal) == 1:
+        integer = integer_decimal[0]
+        decimal = ''
+    elif len(integer_decimal) == 2:
+        integer, decimal = integer_decimal
+    else:
+        raise ValueError(
+            f"The value string: '${value_string}' has more than one point in it."
+        )
+
+    result = verbalize_cardinal(integer)
+
+    decimal = decimal.rstrip('0')
+    if decimal:
+        # '.22' is verbalized as '零点二二'
+        # '3.20' is verbalized as '三点二
+        result = result if result else "零"
+        result += '点' + verbalize_digit(decimal)
+    return result