WeTextProcessing 1.0.1__tar.gz → 1.0.3__tar.gz

{WeTextProcessing-1.0.1 → WeTextProcessing-1.0.3}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: WeTextProcessing
-Version: 1.0.1
+Version: 1.0.3
 Summary: WeTextProcessing, including TN & ITN
 Home-page: https://github.com/wenet-e2e/WeTextProcessing
 Author: Zhendong Peng, Xingchen Song
@@ -49,14 +49,34 @@ weitn --text "二点五平方电线"
 Python usage:
 
 ```py
-# tn usage
->>> from tn.chinese.normalizer import Normalizer
->>> normalizer = Normalizer()
->>> normalizer.normalize("2.5平方电线")
-# itn usage
->>> from itn.chinese.inverse_normalizer import InverseNormalizer
->>> invnormalizer = InverseNormalizer()
->>> invnormalizer.normalize("二点五平方电线")
+from itn.chinese.inverse_normalizer import InverseNormalizer
+from tn.chinese.normalizer import Normalizer as ZhNormalizer
+from tn.english.normalizer import Normalizer as EnNormalizer
+
+# NOTE(xcsong): 和默认参数不一致时，必须重新构图，要重新构图请务必指定 `overwrite_cache=True`
+#               When the parameters differ from the defaults, it is mandatory to re-compose. To re-compose, please ensure you specify `overwrite_cache=True`.
+
+zh_tn_text = "你好 WeTextProcessing 1.0，船新版本儿，船新体验儿，简直666，9和10"
+zh_itn_text = "你好 WeTextProcessing 一点零，船新版本儿，船新体验儿，简直六六六，九和六"
+en_tn_text = "Hello WeTextProcessing 1.0, life is short, just use wetext, 666, 9 and 10"
+zh_tn_model = ZhNormalizer(remove_erhua=True, overwrite_cache=True)
+zh_itn_model = InverseNormalizer(enable_0_to_9=False, overwrite_cache=True)
+en_tn_model = EnNormalizer(overwrite_cache=True)
+print("中文 TN (去除儿化音，重新在线构图):\n\t{} => {}".format(zh_tn_text, zh_tn_model.normalize(zh_tn_text)))
+print("中文ITN (小于10的单独数字不转换，重新在线构图):\n\t{} => {}".format(zh_itn_text, zh_itn_model.normalize(zh_itn_text)))
+print("英文 TN (暂时还没有可控的选项，后面会加...):\n\t{} => {}\n".format(en_tn_text, en_tn_model.normalize(en_tn_text)))
+
+zh_tn_model = ZhNormalizer(overwrite_cache=False)
+zh_itn_model = InverseNormalizer(overwrite_cache=False)
+en_tn_model = EnNormalizer(overwrite_cache=False)
+print("中文 TN (复用之前编译好的图):\n\t{} => {}".format(zh_tn_text, zh_tn_model.normalize(zh_tn_text)))
+print("中文ITN (复用之前编译好的图):\n\t{} => {}".format(zh_itn_text, zh_itn_model.normalize(zh_itn_text)))
+print("英文 TN (复用之前编译好的图):\n\t{} => {}\n".format(en_tn_text, en_tn_model.normalize(en_tn_text)))
+
+zh_tn_model = ZhNormalizer(remove_erhua=False, overwrite_cache=True)
+zh_itn_model = InverseNormalizer(enable_0_to_9=True, overwrite_cache=True)
+print("中文 TN (不去除儿化音，重新在线构图):\n\t{} => {}".format(zh_tn_text, zh_tn_model.normalize(zh_tn_text)))
+print("中文ITN (小于10的单独数字也进行转换，重新在线构图):\n\t{} => {}\n".format(zh_itn_text, zh_itn_model.normalize(zh_itn_text)))
 ```
 
 #### 1.2 Advanced Usage:

{WeTextProcessing-1.0.1 → WeTextProcessing-1.0.3}/README.md

@@ -34,14 +34,34 @@ weitn --text "二点五平方电线"
 Python usage:
 
 ```py
-# tn usage
->>> from tn.chinese.normalizer import Normalizer
->>> normalizer = Normalizer()
->>> normalizer.normalize("2.5平方电线")
-# itn usage
->>> from itn.chinese.inverse_normalizer import InverseNormalizer
->>> invnormalizer = InverseNormalizer()
->>> invnormalizer.normalize("二点五平方电线")
+from itn.chinese.inverse_normalizer import InverseNormalizer
+from tn.chinese.normalizer import Normalizer as ZhNormalizer
+from tn.english.normalizer import Normalizer as EnNormalizer
+
+# NOTE(xcsong): 和默认参数不一致时，必须重新构图，要重新构图请务必指定 `overwrite_cache=True`
+#               When the parameters differ from the defaults, it is mandatory to re-compose. To re-compose, please ensure you specify `overwrite_cache=True`.
+
+zh_tn_text = "你好 WeTextProcessing 1.0，船新版本儿，船新体验儿，简直666，9和10"
+zh_itn_text = "你好 WeTextProcessing 一点零，船新版本儿，船新体验儿，简直六六六，九和六"
+en_tn_text = "Hello WeTextProcessing 1.0, life is short, just use wetext, 666, 9 and 10"
+zh_tn_model = ZhNormalizer(remove_erhua=True, overwrite_cache=True)
+zh_itn_model = InverseNormalizer(enable_0_to_9=False, overwrite_cache=True)
+en_tn_model = EnNormalizer(overwrite_cache=True)
+print("中文 TN (去除儿化音，重新在线构图):\n\t{} => {}".format(zh_tn_text, zh_tn_model.normalize(zh_tn_text)))
+print("中文ITN (小于10的单独数字不转换，重新在线构图):\n\t{} => {}".format(zh_itn_text, zh_itn_model.normalize(zh_itn_text)))
+print("英文 TN (暂时还没有可控的选项，后面会加...):\n\t{} => {}\n".format(en_tn_text, en_tn_model.normalize(en_tn_text)))
+
+zh_tn_model = ZhNormalizer(overwrite_cache=False)
+zh_itn_model = InverseNormalizer(overwrite_cache=False)
+en_tn_model = EnNormalizer(overwrite_cache=False)
+print("中文 TN (复用之前编译好的图):\n\t{} => {}".format(zh_tn_text, zh_tn_model.normalize(zh_tn_text)))
+print("中文ITN (复用之前编译好的图):\n\t{} => {}".format(zh_itn_text, zh_itn_model.normalize(zh_itn_text)))
+print("英文 TN (复用之前编译好的图):\n\t{} => {}\n".format(en_tn_text, en_tn_model.normalize(en_tn_text)))
+
+zh_tn_model = ZhNormalizer(remove_erhua=False, overwrite_cache=True)
+zh_itn_model = InverseNormalizer(enable_0_to_9=True, overwrite_cache=True)
+print("中文 TN (不去除儿化音，重新在线构图):\n\t{} => {}".format(zh_tn_text, zh_tn_model.normalize(zh_tn_text)))
+print("中文ITN (小于10的单独数字也进行转换，重新在线构图):\n\t{} => {}\n".format(zh_itn_text, zh_itn_model.normalize(zh_itn_text)))
 ```
 
 #### 1.2 Advanced Usage:

{WeTextProcessing-1.0.1 → WeTextProcessing-1.0.3}/WeTextProcessing.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: WeTextProcessing
-Version: 1.0.1
+Version: 1.0.3
 Summary: WeTextProcessing, including TN & ITN
 Home-page: https://github.com/wenet-e2e/WeTextProcessing
 Author: Zhendong Peng, Xingchen Song
@@ -49,14 +49,34 @@ weitn --text "二点五平方电线"
 Python usage:
 
 ```py
-# tn usage
->>> from tn.chinese.normalizer import Normalizer
->>> normalizer = Normalizer()
->>> normalizer.normalize("2.5平方电线")
-# itn usage
->>> from itn.chinese.inverse_normalizer import InverseNormalizer
->>> invnormalizer = InverseNormalizer()
->>> invnormalizer.normalize("二点五平方电线")
+from itn.chinese.inverse_normalizer import InverseNormalizer
+from tn.chinese.normalizer import Normalizer as ZhNormalizer
+from tn.english.normalizer import Normalizer as EnNormalizer
+
+# NOTE(xcsong): 和默认参数不一致时，必须重新构图，要重新构图请务必指定 `overwrite_cache=True`
+#               When the parameters differ from the defaults, it is mandatory to re-compose. To re-compose, please ensure you specify `overwrite_cache=True`.
+
+zh_tn_text = "你好 WeTextProcessing 1.0，船新版本儿，船新体验儿，简直666，9和10"
+zh_itn_text = "你好 WeTextProcessing 一点零，船新版本儿，船新体验儿，简直六六六，九和六"
+en_tn_text = "Hello WeTextProcessing 1.0, life is short, just use wetext, 666, 9 and 10"
+zh_tn_model = ZhNormalizer(remove_erhua=True, overwrite_cache=True)
+zh_itn_model = InverseNormalizer(enable_0_to_9=False, overwrite_cache=True)
+en_tn_model = EnNormalizer(overwrite_cache=True)
+print("中文 TN (去除儿化音，重新在线构图):\n\t{} => {}".format(zh_tn_text, zh_tn_model.normalize(zh_tn_text)))
+print("中文ITN (小于10的单独数字不转换，重新在线构图):\n\t{} => {}".format(zh_itn_text, zh_itn_model.normalize(zh_itn_text)))
+print("英文 TN (暂时还没有可控的选项，后面会加...):\n\t{} => {}\n".format(en_tn_text, en_tn_model.normalize(en_tn_text)))
+
+zh_tn_model = ZhNormalizer(overwrite_cache=False)
+zh_itn_model = InverseNormalizer(overwrite_cache=False)
+en_tn_model = EnNormalizer(overwrite_cache=False)
+print("中文 TN (复用之前编译好的图):\n\t{} => {}".format(zh_tn_text, zh_tn_model.normalize(zh_tn_text)))
+print("中文ITN (复用之前编译好的图):\n\t{} => {}".format(zh_itn_text, zh_itn_model.normalize(zh_itn_text)))
+print("英文 TN (复用之前编译好的图):\n\t{} => {}\n".format(en_tn_text, en_tn_model.normalize(en_tn_text)))
+
+zh_tn_model = ZhNormalizer(remove_erhua=False, overwrite_cache=True)
+zh_itn_model = InverseNormalizer(enable_0_to_9=True, overwrite_cache=True)
+print("中文 TN (不去除儿化音，重新在线构图):\n\t{} => {}".format(zh_tn_text, zh_tn_model.normalize(zh_tn_text)))
+print("中文ITN (小于10的单独数字也进行转换，重新在线构图):\n\t{} => {}\n".format(zh_itn_text, zh_itn_model.normalize(zh_itn_text)))
 ```
 
 #### 1.2 Advanced Usage:

{WeTextProcessing-1.0.1 → WeTextProcessing-1.0.3}/WeTextProcessing.egg-info/SOURCES.txt

@@ -25,6 +25,7 @@ itn/chinese/data/measure/units_zh.tsv
 itn/chinese/data/money/code.tsv
 itn/chinese/data/money/symbol.tsv
 itn/chinese/data/number/digit.tsv
+itn/chinese/data/number/digit_zh.tsv
 itn/chinese/data/number/dot.tsv
 itn/chinese/data/number/sign.tsv
 itn/chinese/data/number/special_dash.tsv

{WeTextProcessing-1.0.1 → WeTextProcessing-1.0.3}/itn/chinese/data/default/whitelist.tsv

@@ -61,3 +61,18 @@
 五台山	五台山
 六盘水	六盘水
 八宿	八宿
+十二五	十二五
+十三五	十三五
+十四五	十四五
+几十万	几十万
+几百万	几百万
+几千万	几千万
+十几万	十几万
+二十几万	二十几万
+三十几万	三十几万
+四十几万	四十几万
+五十几万	五十几万
+六十几万	六十几万
+七十几万	七十几万
+八十几万	八十几万
+九十几万	九十几万

WeTextProcessing-1.0.3/itn/chinese/data/number/digit_zh.tsv

@@ -0,0 +1,10 @@
+一
+二
+两
+三
+四
+五
+六
+七
+八
+九

{WeTextProcessing-1.0.1 → WeTextProcessing-1.0.3}/itn/chinese/rules/cardinal.py

@@ -69,7 +69,7 @@ class Cardinal(Processor):
                                           | add_weight(addzero**2, 1.0)))
         # 一千一百一十一 => 1111, 一千零一十一 => 1011, 一千零一 => 1001
         # 一千一 => 1100, 一千 => 1000
-        thousand = ((hundred | teen | tens | digits) + delete('千') +
+        thousand = (digit + delete('千') +
                     (hundred
                      | add_weight(zero + (tens | teen), 0.1)
                      | add_weight(addzero + zero + digit, 0.5)
@@ -78,7 +78,7 @@ class Cardinal(Processor):
         # 10001111, 1001111, 101111, 11111, 10111, 10011, 10001, 10000
         if self.enable_million:
             ten_thousand = (
-                (thousand | hundred | teen | tens | digits) + delete('万') +
+                (thousand | hundred | teen | tens | digit) + delete('万') +
                 (thousand
                  | add_weight(zero + hundred, 0.1)
                  | add_weight(addzero + zero + (tens | teen), 0.5)
@@ -87,7 +87,7 @@ class Cardinal(Processor):
                  | add_weight(addzero**4, 1.0)))
         else:
             ten_thousand = (
-                (teen | tens | digits) + delete('万') +
+                (teen | tens | digit) + delete('万') +
                 (thousand
                  | add_weight(zero + hundred, 0.1)
                  | add_weight(addzero + zero + (tens | teen), 0.5)

{WeTextProcessing-1.0.1 → WeTextProcessing-1.0.3}/itn/chinese/rules/measure.py

@@ -16,7 +16,7 @@ from itn.chinese.rules.cardinal import Cardinal
 from tn.processor import Processor
 from tn.utils import get_abs_path
 
-from pynini import string_file, accep, cross
+from pynini import string_file, accep, cross, union
 from pynini.lib.pynutil import delete, insert, add_weight
 
 
@@ -36,6 +36,11 @@ class Measure(Processor):
             get_abs_path('../itn/chinese/data/measure/units_zh.tsv'))
         sign = string_file(
             get_abs_path('../itn/chinese/data/number/sign.tsv'))  # + -
+        digit = string_file(
+            get_abs_path('../itn/chinese/data/number/digit.tsv'))  # 1 ~ 9
+        digit_zh = string_file(
+            get_abs_path('../itn/chinese/data/number/digit_zh.tsv'))  # 1 ~ 9
+        addzero = insert('0')
         to = cross('到', '~') | cross('到百分之', '~')
 
         units = add_weight(
@@ -55,8 +60,35 @@ class Measure(Processor):
 
         # 十千米每小时 => 10km/h, 十一到一百千米每小时 => 11~100km/h
         measure = number + (to + number).ques + units
-        tagger = insert('value: "') + (measure | percent) + insert('"')
 
+        # XXX: 特殊case处理, ignore enable_standalone_number
+        # digit + union("百", "千", "万") + digit + unit
+        unit_sp_case1 = [
+            '年',
+            '月',
+            '个月',
+            '周',
+            '天',
+            '位',
+            '次',
+            '个',
+            '顿',
+        ]
+        if self.enable_0_to_9:
+            measure_sp = add_weight(
+                ((digit + delete('百') + add_weight(addzero**2, 1.0)) |
+                 (digit + delete('千') + add_weight(addzero**3, 1.0)) |
+                 (digit + delete('万') + add_weight(addzero**4, 1.0))) +
+                insert(' ') + digit + union(*unit_sp_case1), -0.5)
+        else:
+            measure_sp = add_weight(
+                ((digit + delete('百') + add_weight(addzero**2, 1.0)) |
+                 (digit + delete('千') + add_weight(addzero**3, 1.0)) |
+                 (digit + delete('万') + add_weight(addzero**4, 1.0))) +
+                digit_zh + union(*unit_sp_case1), -0.5)
+
+        tagger = insert('value: "') + (measure | measure_sp
+                                       | percent) + insert('"')
         # 每小时十千米 => 10km/h, 每小时三十到三百一十一千米 => 30~311km/h
         tagger |= (insert('denominator: "') + delete('每') + units +
                    insert('" numerator: "') + measure + insert('"'))

{WeTextProcessing-1.0.1 → WeTextProcessing-1.0.3}/tn/chinese/data/char/punctuations_zh.tsv

@@ -12,6 +12,7 @@
 、
 。
 〃
+《
 》
 「
 」

{WeTextProcessing-1.0.1 → WeTextProcessing-1.0.3}/tn/chinese/data/char/traditional_to_simple.tsv

@@ -3143,3 +3143,4 @@
 龔	龚
 龕	龛
 龜	龟
+後	后

{WeTextProcessing-1.0.1 → WeTextProcessing-1.0.3}/tn/english/data/whitelist/tts.tsv

@@ -3061,3 +3061,9 @@ Z. Y.	ZY
 Z.Y.	ZY
 Z. Z.	ZZ
 Z.Z.	ZZ
+here's	here is
+you're	you are
+i'm	i am
+Here's	Here is
+You're	You are
+I'm	I am

{WeTextProcessing-1.0.1 → WeTextProcessing-1.0.3}/tn/english/normalizer.py

@@ -57,15 +57,25 @@ class Normalizer(Processor):
         punct = add_weight(Punctuation().tagger, 2.00)
         rang = add_weight(Range().tagger, 1.01)
         # TODO(xcsong): add roman
-        tagger = punct.star + \
-            (cardinal | ordinal | word
-             | date | decimal | fraction
-             | time | measure | money
-             | telephone | electronic
+        tagger = \
+            (cardinal
+             | ordinal
+             | word
+             | date
+             | decimal
+             | fraction
+             | time
+             | measure
+             | money
+             | telephone
+             | electronic
              | whitelist
-             | rang).optimize() + (punct.plus | self.DELETE_SPACE)
-        # delete the last space
-        self.tagger = tagger.star @ self.build_rule(delete(' '), r='[EOS]')
+             | rang
+             | punct
+             ).optimize() + (punct.plus | self.DELETE_SPACE)
+        # delete the first and last space
+        self.tagger = (delete(' ').star + tagger.star) @ self.build_rule(
+            delete(' '), r='[EOS]')
 
     def build_verbalizer(self):
         cardinal = Cardinal().verbalizer
@@ -83,14 +93,20 @@ class Normalizer(Processor):
         punct = Punctuation().verbalizer
         rang = Range().verbalizer
         verbalizer = \
-            (cardinal | ordinal | word
-             | date | decimal
-             | fraction | time
-             | measure | money
+            (cardinal
+             | ordinal
+             | word
+             | date
+             | decimal
+             | fraction
+             | time
+             | measure
+             | money
              | telephone
              | electronic
              | whitelist
              | punct
-             | rang).optimize() + (punct.plus | self.INSERT_SPACE)
+             | rang
+             ).optimize() + (punct.plus | self.INSERT_SPACE)
         self.verbalizer = verbalizer.star @ self.build_rule(delete(' '),
                                                             r='[EOS]')

{WeTextProcessing-1.0.1 → WeTextProcessing-1.0.3}/tn/english/rules/punctuation.py

@@ -61,14 +61,15 @@ class Punctuation(Processor):
         punct = closure(self.punct | cross('\\', '\\\\\\') | cross('"', '\\"'),
                         1)
 
-        emphasis = (
+        self.emphasis = (
             accep("<") +
             ((
                 closure(self.NOT_SPACE - union("<", ">"), 1) +  # noqa
                 closure(accep("/"), 0, 1))  # noqa
              | (accep("/") + closure(self.NOT_SPACE - union("<", ">"), 1))) +
             accep(">"))  # noqa
-        punct = plurals._priority_union(emphasis, punct, closure(self.VCHAR))
+        punct = plurals._priority_union(self.emphasis, punct,
+                                        closure(self.VCHAR))
 
         self.graph = punct
         final_graph = insert("v: \"") + add_weight(
@@ -78,7 +79,10 @@ class Punctuation(Processor):
 
     def build_verbalizer(self):
         punct = closure(
-            self.punct | cross('\\\\\\', '\\') | cross('\\"', '"')
-            | accep(" "), 1)
-        verbalizer = delete('v: "') + punct + delete('"')
+            self.punct | self.emphasis | cross('\\\\\\', '\\')
+            | cross('\\"', '"'), 1)
+        verbalizer = delete('v: "') + add_weight(accep(" "), -1.0).star \
+            + punct \
+            + add_weight(accep(" "), -1.0).star \
+            + delete('"')
         self.verbalizer = self.delete_tokens(verbalizer)

{WeTextProcessing-1.0.1 → WeTextProcessing-1.0.3}/tn/main.py

@@ -73,9 +73,8 @@ def main():
             full_to_half=str2bool(args.full_to_half),
             tag_oov=str2bool(args.tag_oov))
     elif args.language == "en":
-        normalizer = EnNormalizer(
-            cache_dir=args.cache_dir,
-            overwrite_cache=args.overwrite_cache)
+        normalizer = EnNormalizer(cache_dir=args.cache_dir,
+                                  overwrite_cache=args.overwrite_cache)
 
     if args.text:
         print(normalizer.tag(args.text))