sigilyph 0.5.2__py3-none-any.whl

sigilyph/text_norm/norm_func.py

@@ -0,0 +1,155 @@
+'''
+FilePath: /python-Sigilyph/sigilyph/text_norm/norm_func.py
+Descripttion: 
+Author: Yixiang Chen
+version: 
+Date: 2025-03-31 17:50:26
+LastEditors: Yixiang Chen
+LastEditTime: 2026-01-19 20:05:35
+'''
+
+
+import re
+
+from sigilyph.core.symbols import punctuation, punc_map_ch
+
+from sigilyph.fst_tool.infer_normalizer import ZhNormalizer, EnNormalizer
+
+
+import os
+from importlib_resources import files
+basedir = files('sigilyph')
+
+zh_tn_model = ZhNormalizer(version_id='v2', cache_dir=os.path.join(basedir, 'text_norm', 'cache_dir'), remove_erhua=False, full_to_half=False)
+en_tn_model = EnNormalizer(version_id='v2', cache_dir=os.path.join(basedir, 'text_norm', 'cache_dir'))
+
+import json
+with open(os.path.join(basedir, 'core', 'special_dict.json'), 'r', encoding="utf-8") as infi:
+    special_dict = json.load(infi)
+
+def pro_norm(text, use_lang='zh'):
+    if use_lang == 'zh':
+        norm_text = zh_tn_model.normalize(text)
+        #print("zh ", norm_text)
+    else:
+        norm_text = en_tn_model.normalize(text)
+        #print("en ", norm_text)
+    return norm_text
+
+def replace_with_dict(text, replace_dict):
+    for old, new in replace_dict.items():
+        text = text.replace(old, new)
+    return text
+
+def replace_with_dict_re(text, replace_dict):
+    pattern = re.compile("|".join(re.escape(key) for key in replace_dict.keys()))
+    return pattern.sub(lambda m: replace_dict[m.group(0)], text)
+
+def replace_roman_1_to_10(text: str) -> str:
+    """
+    将字符串中的罗马数字符号 Ⅰ～Ⅹ 替换为中文数字 一～十。
+    其余任何字符（包括乱码、英文 I/V/X 等）都保持不变。
+    """
+    roman_to_cn = {
+        'Ⅰ': '一',
+        'Ⅱ': '二',
+        'Ⅲ': '三',
+        'Ⅳ': '四',
+        'Ⅴ': '五',
+        'Ⅵ': '六',
+        'Ⅶ': '七',
+        'Ⅷ': '八',
+        'Ⅸ': '九',
+        'Ⅹ': '十',
+    }
+    # 逐字符扫描，能映射的就换成中文数字，不能映射的原样保留
+    return ''.join(roman_to_cn.get(ch, ch) for ch in text)
+
+pre_replace_dict = {"AlphaFold-Plus": "AlphaFold Plus"}
+def preprocess_first_old(text, use_lang='zh'):
+    text = replace_with_dict(text, pre_replace_dict)
+    norm_text = pro_norm(text, use_lang)
+    #print(norm_text)
+    rep_text = replace_with_dict(norm_text, special_dict)
+    return rep_text
+
+def preprocess_first(text, before_replace_dict, special_word_dict, norm_use_lang='zh'):
+    text = replace_with_dict(text, before_replace_dict)
+    norm_text = pro_norm(text, norm_use_lang)
+    #print(norm_text)
+    #rep_text = replace_with_dict(norm_text, special_word_dict)
+    return norm_text
+def post_process(text , special_word_dict):
+    rep_text = replace_with_dict(text, special_word_dict)
+    return rep_text
+
+
+def preprocess_first_for_norm(text, before_replace_dict, norm_use_lang='zh'):
+    text = replace_roman_1_to_10(text)
+    text = replace_with_dict(text, before_replace_dict)
+    return text
+
+def normalizer(text):
+    return text
+
+def replace_punc(text):
+    #text = text.replace("嗯", "恩").replace("呣", "母")
+    pattern = re.compile("|".join(re.escape(p) for p in punc_map_ch.keys()))
+    replaced_text = pattern.sub(lambda x: punc_map_ch[x.group()], text)
+    replaced_text = re.sub(
+        r"[^\u4e00-\u9fa5" + "".join(punctuation) + r"]+", "", replaced_text
+    )
+    return replaced_text
+
+def replace_punc_part(text: str) -> str:
+    """
+    将中文句号、逗号、分号、引号替换为英文符号：
+    。 -> .
+    ， -> ,
+    ； -> ;
+    “ -> "
+    ” -> "
+    『 -> "
+    』 -> "
+    「 -> "
+    」 -> "
+    『』、「」等都统一为英文双引号
+    其他符号不动
+    """
+    # 建立映射表
+    mapping = {
+        '。': '.',
+        '，': ',',
+        '；': ';',
+        '“': '"',
+        '”': '"',
+        '「': '"',
+        '」': '"',
+        '『': '"',
+        '』': '"',
+    }
+
+    # 构造正则：匹配所有需要替换的中文标点
+    pattern = re.compile(r'[。，“”；「」『』]')
+
+    # 使用 re.sub 进行替换
+    return pattern.sub(lambda m: mapping[m.group(0)], text)
+
+'''
+def text_norm_cn(text):
+    text = normalizer(text)
+    text = replace_punc(text)
+    return text
+
+def text_norm_en(text):
+    return text 
+'''
+
+def text_norm_cn(text):
+    norm_text = zh_tn_model.normalize(text) 
+    norm_text = replace_punc_part(norm_text)
+    return norm_text
+
+def text_norm_en(text):
+    norm_text = en_tn_model.normalize(text)
+    return norm_text 

sigilyph/text_norm/norm_func_new.py

@@ -0,0 +1,89 @@
+'''
+FilePath: /python-Sigilyph/sigilyph/text_norm/norm_func.py
+Descripttion: 
+Author: Yixiang Chen
+version: 
+Date: 2025-03-31 17:50:26
+LastEditors: Yixiang Chen
+LastEditTime: 2026-01-19 10:09:56
+'''
+
+import os
+import json
+import re
+from importlib_resources import files
+from sigilyph.core.symbols import punctuation, punc_map_ch
+from sigilyph.fst_tool.infer_normalizer import ZhNormalizer, EnNormalizer
+
+basedir = files('sigilyph')
+zh_tn_model = ZhNormalizer(version_id='v2', cache_dir=os.path.join(basedir, 'text_norm', 'cache_dir'), remove_erhua=False, full_to_half=False)
+en_tn_model = EnNormalizer(version_id='v2', cache_dir=os.path.join(basedir, 'text_norm', 'cache_dir'))
+
+#with open(os.path.join(basedir, 'core', 'special_dict.json'), 'r', encoding="utf-8") as infi:
+#    special_dict = json.load(infi)
+
+def pro_norm(text, use_lang='zh'):
+    """Normalize text based on the specified language."""
+    if use_lang == 'zh':
+        return zh_tn_model.normalize(text)
+    return en_tn_model.normalize(text)
+
+def replace_with_dict(text, replace_dict):
+    """Replace occurrences of keys in text with their corresponding values from replace_dict."""
+    for old, new in replace_dict.items():
+        text = text.replace(old, new)
+    return text
+
+def replace_with_dict_re(text, replace_dict):
+    """Replace occurrences of keys in text using regular expressions."""
+    pattern = re.compile("|".join(re.escape(key) for key in replace_dict.keys()))
+    return pattern.sub(lambda m: replace_dict[m.group(0)], text)
+
+def preprocess_first(text, before_replace_dict, special_word_dict, norm_use_lang='zh'):
+    """Preprocess text by replacing specified words and normalizing."""
+    text = replace_with_dict(text, before_replace_dict)
+    norm_text = pro_norm(text, norm_use_lang)
+    return replace_with_dict(norm_text, special_word_dict)
+
+def post_process(text, special_word_dict):
+    """Post-process text by replacing special words."""
+    return replace_with_dict(text, special_word_dict)
+
+def preprocess_first_for_norm(text, before_replace_dict):
+    """Preprocess text for normalization."""
+    return replace_with_dict(text, before_replace_dict)
+
+def normalizer(text):
+    """Placeholder for a normalizer function."""
+    return text
+
+def replace_punc(text):
+    """Replace Chinese punctuation with corresponding characters."""
+    pattern = re.compile("|".join(re.escape(p) for p in punc_map_ch.keys()))
+    replaced_text = pattern.sub(lambda x: punc_map_ch[x.group()], text)
+    return re.sub(r"[^\u4e00-\u9fa5" + "".join(punctuation) + r"]+", "", replaced_text)
+
+def replace_punc_part(text: str) -> str:
+    """Replace specific Chinese punctuation with English punctuation."""
+    mapping = {
+        '。': '.',
+        '，': ',',
+        '；': ';',
+        '“': '"',
+        '”': '"',
+        '「': '"',
+        '」': '"',
+        '『': '"',
+        '』': '"',
+    }
+    pattern = re.compile(r'[。，“”；「」『』]')
+    return pattern.sub(lambda m: mapping[m.group(0)], text)
+
+def text_norm_cn(text):
+    """Normalize Chinese text."""
+    norm_text = zh_tn_model.normalize(text)
+    return replace_punc_part(norm_text)
+
+def text_norm_en(text):
+    """Normalize English text."""
+    return en_tn_model.normalize(text)

sigilyph/text_norm/sigilyph_norm.py

@@ -0,0 +1,179 @@
+'''
+FilePath: /python-Sigilyph/sigilyph/text_norm/sigilyph_norm.py
+Descripttion: 
+Author: Yixiang Chen
+version: 
+Date: 2026-01-07 15:46:04
+LastEditors: Yixiang Chen
+LastEditTime: 2026-01-19 19:55:00
+'''
+
+import langid
+import re
+import jieba
+import os
+
+from sigilyph.text_norm.norm_func import preprocess_first_for_norm, text_norm_en, text_norm_cn
+from sigilyph.core.predict import special_phrase
+
+norm_func_dict = {
+    'en': text_norm_en,
+    'zh': text_norm_cn
+}
+
+import json
+from importlib_resources import files
+basedir = files('sigilyph')
+with open(os.path.join(basedir, 'text_norm', 'dict_special_word_polyphone.json'), 'r', encoding="utf-8") as infi:
+    dict_special_word_polyphone_json = json.load(infi)
+    dict_special_word_polyphone = dict_special_word_polyphone_json['polyphone_config']
+with open(os.path.join(basedir, 'text_norm', 'dict_special_word_base.json'), 'r', encoding="utf-8") as infib:
+    dict_special_word_base_json = json.load(infib)
+    dict_special_word_base = dict_special_word_base_json['base_config']
+
+def is_float_strip(s: str) -> bool:
+    s = s.strip()   # 只去掉首尾空白
+    if not s:
+        return False
+    try:
+        float(s)
+        return True
+    except ValueError:
+        return False
+
+class SigilyphNormalizer:
+    def __init__(self, norm_use_dict) -> None:
+        self.sil1symbol='-'
+        self.special_phrase = special_phrase 
+
+        self.base_replace_dict = dict_special_word_base
+        self.base_replace_dict.update(dict_special_word_polyphone)
+
+        self.before_replace_dict = self.base_replace_dict
+        self.before_replace_dict.update(norm_use_dict)
+    
+    def fix_replace_dict(self, new_before_replace_dict):
+        self.before_replace_dict = self.base_replace_dict
+        self.before_replace_dict.update(new_before_replace_dict)
+
+    def normalize(self, text, lang, norm_use_lang='zh'):
+        text = preprocess_first_for_norm(text, self.before_replace_dict, norm_use_lang=norm_use_lang)
+        multi_lang_text_list = self.text_split_lang(text, lang) 
+        all_phone = []
+        outtext = ''
+        for text_split_dict in multi_lang_text_list:
+            use_lang = text_split_dict['lang']
+            use_text = text_split_dict['text_split']
+            if use_lang not in norm_func_dict.keys():
+                use_lang = 'zh'
+            use_text = self.text_norm(use_text, use_lang)
+            outtext += use_text 
+        return outtext 
+    
+    ######## text norm #########
+    def text_norm(self, text, lang):
+        outtext = norm_func_dict[lang](text)
+        return outtext
+    
+    def split_with_units(self, text, regex):
+        # 中文数字（常见大写+小写口语化）
+        CHINESE_NUM_CHARS = "零一二三四五六七八九十百千万亿两〇壹贰叁肆伍陆柒捌玖拾佰仟萬億"
+        # 单位模式：可按需要继续扩展
+        unit_pattern = re.compile(r'^(\s*)(km/h|km|m/s|m|s|g|h|kg|mm|cm)\b')
+        
+        pieces = re.findall(regex, text)
+        merged = []
+
+        for piece in pieces:
+            if merged:
+                m = unit_pattern.match(piece)
+                if m:
+                    # 前一块最后一个字符
+                    last_char = merged[-1][-1]
+                    # 条件：前一块以“汉字或阿拉伯数字或中文数字”结尾
+                    if (
+                        re.match(r'[\u4e00-\u9fff\u3400-\u4dbf0-9]', last_char)
+                        or last_char in CHINESE_NUM_CHARS
+                    ):
+                        # 把单位并到前一块
+                        merged[-1] += m.group(1) + m.group(2)
+                        # 当前块剩余部分（若有）单独保留
+                        rest = piece[m.end():]
+                        if rest:
+                            merged.append(rest)
+                        continue
+
+            merged.append(piece)
+
+        return merged
+
+    ###############  split text in line with lang ##############
+    def text_split_lang(self, text, lang):
+        if lang == 'ZH' or lang == 'zh':
+            multi_lang_text_list = [{'lang':'zh', 'text_split': text}]
+        elif lang == 'en':
+            multi_lang_text_list = [{'lang':'en', 'text_split': text}]
+        else:
+            # Phoneme be judged
+            pretext_split =  re.split("(\[.*?\])", text, re.I|re.M)
+            multi_lang_text_list = []
+            pretext_split = list(filter(None, pretext_split))
+            for utext in pretext_split:
+                if utext[0] != '[':
+                    #pattern = r'([a-zA-Z ,.\!\?]+|[\u4e00-\u9fa5 ，。,.\t \"\！\？\“\”\、]+)'
+                    #text_split = re.findall(pattern, utext)
+                    pattern = r'''(
+                        # ---------- 中文块 ----------
+                        # 汉字 + 数字 + 日期时间符号 + 中/英逗号句号 + 常见中文标点 +
+                        # 全角空格 + 半角空格 + ℃ + / + % + 单位字母 k,m,g,h（大小写都算）
+                        [\u4e00-\u9fff\u3400-\u4dbf
+                        0-9
+                        \-:~_                      # 日期/时间里的 - 和 :
+                        ，。！？：；、…“”‘’「」『』《》．【】（）\u3000
+                        ,\.                      # 英文逗号、英文句号
+                        \x20                     # 半角空格
+                        /%                       # / 和 %
+                        ℃
+                        $£￡¥￥฿€₹₽CHFR$
+                        ]+
+                        |
+                        # ---------- 英文块 ----------
+                        # 字母 + 数字 + 英文标点 + 其它空白（制表符/换行等）
+                        [a-zA-Z
+                        ,\.!?;:'"\-\(\)\[\]/\\_@#\$%&\+
+                        \t\r\n\f\v               # 其它空白（不含普通空格）
+                        ]+
+                        |
+                        # ---------- 其它块 ----------
+                        # 不属于上面两类的字符（emoji、特殊符号等）
+                        [^a-zA-Z0-9
+                        \u4e00-\u9fff\u3400-\u4dbf
+                        ，。！？：；、…“”‘’「」『』《》．【】（）\u3000
+                        \-:
+                        ,\.
+                        \x20\t\r\n\f\v
+                        /%
+                        ℃
+                        ]+
+                    )'''
+                    regex = re.compile(pattern, re.VERBOSE)
+                    #text_split = re.findall(regex, utext)
+                    text_split = self.split_with_units(utext, regex)
+                    for idx in range(len(text_split)):
+                        tmpts = text_split[idx]
+                        #if tmpts.strip().isdigit():
+                        if is_float_strip(tmpts):
+                            tmp_lang = 'zh'
+                        else:
+                            tmp_lang = langid.classify(tmpts)[0]
+                        if tmp_lang in ['zh', 'jp', 'ja']:
+                            tmp_lang = 'zh'
+                            #tmpts = tmpts.replace(' ', self.sil1symbol)
+                        else:
+                            tmp_lang = 'en' 
+                        multi_lang_text_list.append({'lang':tmp_lang, 'text_split': tmpts})
+                else:
+                    phones = utext[1:-1]
+                    multi_lang_text_list.append({'lang':'phone', 'text_split': phones})
+        return multi_lang_text_list
+

sigilyph-0.5.2.dist-info/METADATA

@@ -0,0 +1,24 @@
+Metadata-Version: 2.1
+Name: sigilyph
+Version: 0.5.2
+Summary: Text Front for TTS
+Home-page: https://github.com/yixiangchen1995/python-Sigilyph
+Author: Yixiang Chen
+Author-email: <yixiangchen1995@gmail.com>
+License: MIT
+Keywords: python,first package
+Classifier: Development Status :: 3 - Alpha
+Classifier: Intended Audience :: Developers
+Classifier: Programming Language :: Python
+Classifier: Programming Language :: Python :: 3.10
+Description-Content-Type: text/markdown
+Requires-Dist: g2p-en
+Requires-Dist: jieba
+Requires-Dist: jieba-fast
+Requires-Dist: pypinyin
+Requires-Dist: pynini
+Requires-Dist: importlib-resources
+Requires-Dist: langid
+
+# python-Sigilyph
+The TTS Text Frontend for the use of own

sigilyph-0.5.2.dist-info/RECORD

@@ -0,0 +1,24 @@
+sigilyph/__init__.py,sha256=sUTS6j9YR8pPW0US_QzS-jWYS0ztrt6AiAhkUJARgTo,403
+sigilyph/core/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+sigilyph/core/bert_align.py,sha256=mIzqch3c-skHU9VmyNLjy-bRUt9eD8SqbAUXOZdaR-I,5741
+sigilyph/core/g2p_func.py,sha256=ccwnMA9VgK0xTWKPVR1Pwb5Gop-CNH5_ipHH8SupNUk,1235
+sigilyph/core/norm_func_bk.py,sha256=GRxsloM7mbGFVsLz9gtSA1-W6G9Ij3hzGvcuxe8q6As,3365
+sigilyph/core/predict.py,sha256=ZQZIjui7eHs8MteqBa5tJMW0nyFRfITPfH_xrmHwQZ0,1808
+sigilyph/core/preprocess.py,sha256=l55mqh6TK2cuFQhcl8ouys1-U9XfgNFwXLpFeKI17xk,316
+sigilyph/core/py2phone.dict,sha256=RdafObGTAX9OxvHkijDEzsvyvXnsJuY7aPpBKHgic9g,24894
+sigilyph/core/sigilyph_class.py,sha256=jmKtJf0QQK2N5_gHjTmV0T8y38EqnSp28ardm4xHl3k,9398
+sigilyph/core/special_dict.json,sha256=LtFVDr6OnW8aLCs3z3IzU6rs57aoW8QAiOcqc2bXwys,899
+sigilyph/core/symbols.py,sha256=4IHCCRTiKIaWdMmrUozELlDgkWHAo7l1tPT70Lc_5-w,53898
+sigilyph/core/text_process.py,sha256=ZmttWo89i43aaq30R6Ww2mqcl6O6EYiVP95Y9zOyr7A,11821
+sigilyph/fst_tool/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+sigilyph/fst_tool/infer_normalizer.py,sha256=g8KCFGTwg3AI4gJ7Fb7o3hTStA_0i3q6cqQ6-7-KkAc,2008
+sigilyph/fst_tool/processor.py,sha256=WtiTB7ZCqeUfTFlEhIK_xfBdfO2TsH5LFr5NIFYSR1Y,4735
+sigilyph/fst_tool/token_parser.py,sha256=vdI0_QAq4O1tRjsYQhxouehaqK1xt_VL84v2ikc3l5w,4494
+sigilyph/text_norm/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+sigilyph/text_norm/norm_func.py,sha256=hBCKBxaKNpKtPw4e3iGvFDweSxVu6M4LHW4c6cwM7Hw,4472
+sigilyph/text_norm/norm_func_new.py,sha256=K3Xzc9C9FeNc0vWafFbZYo1zYldV9IzlTWwcC9gcprM,3247
+sigilyph/text_norm/sigilyph_norm.py,sha256=gbRzJu_Pzm6f_IkA6_n0g4K-FaIn_4gY7NhXDAg9JUk,7572
+sigilyph-0.5.2.dist-info/METADATA,sha256=lNb9ZIPrHXimAru0j4PoROZNGYYUKAojdJ-eJ_vWjW0,703
+sigilyph-0.5.2.dist-info/WHEEL,sha256=GJ7t_kWBFywbagK5eo9IoUwLW6oyOeTKmQ-9iHFVNxQ,92
+sigilyph-0.5.2.dist-info/top_level.txt,sha256=caZwwDzakMbaNNk9MOK172HjSUj1HmJr3oK4iOGUyTo,9
+sigilyph-0.5.2.dist-info/RECORD,,

sigilyph-0.5.2.dist-info/WHEEL

@@ -0,0 +1,5 @@
+Wheel-Version: 1.0
+Generator: bdist_wheel (0.43.0)
+Root-Is-Purelib: true
+Tag: py3-none-any
+

sigilyph-0.5.2.dist-info/top_level.txt

@@ -0,0 +1 @@
+sigilyph