WeTextProcessing 0.1.11__py3-none-any.whl → 0.2.0__py3-none-any.whl

{WeTextProcessing-0.1.11.dist-info → WeTextProcessing-0.2.0.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: WeTextProcessing
-Version: 0.1.11
+Version: 0.2.0
 Summary: WeTextProcessing, including TN & ITN
 Home-page: https://github.com/wenet-e2e/WeTextProcessing
 Author: Zhendong Peng, Xingchen Song

WeTextProcessing-0.2.0.dist-info/RECORD

@@ -0,0 +1,112 @@
+itn/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+itn/__main__.py,sha256=jFh8bEB_lC5y7ab9VMqAxWRaI8D-SpBSTiHfMJC1b10,65
+itn/main.py,sha256=NnT9Sh3VTEI0sX_DO9IGVsXCWtZyfRbzj9Sj_6cBxqM,2610
+itn/zh_itn_tagger.fst,sha256=vRgLHTqqchvubK6kKt8_fHhbxRfBykCbLQfhFMk_I64,1080610
+itn/zh_itn_verbalizer.fst,sha256=pHXRUfzls2UmWwTmHHhYmBWMusIfWpU3I7IwxeRjn28,119018
+itn/chinese/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+itn/chinese/inverse_normalizer.py,sha256=HCNW5WCHEYeu60f8lxRmvpXh_48GcXa036EFFR9PQo0,3560
+itn/chinese/rules/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+itn/chinese/rules/cardinal.py,sha256=x3pBfnqzWr85BMkYSOJ87IPL_vAqrCGrUDJvugbq5ZY,7385
+itn/chinese/rules/char.py,sha256=wWnTfzJqE5j7kvEao9rKSN29OVFGNnL25MxS7PcfWEM,965
+itn/chinese/rules/date.py,sha256=dA_AQTXaKfO9U8Mtp-ZT9bYdsaOhp5GHC1UpArNm5DY,2319
+itn/chinese/rules/fraction.py,sha256=cYfBcV3XrXo7SIKFOfvOKjFITnhqxJkyiEaU8byiI8A,1966
+itn/chinese/rules/license_plate.py,sha256=DX9JJjQhh8L6G_8fkVWuabABtw5wjFwQdfS3zxili6M,1439
+itn/chinese/rules/math.py,sha256=WmcZQdns7pY4v5gQajEacnHbedP3w_KJVn3WUvoeZkE,1199
+itn/chinese/rules/measure.py,sha256=JJSdhporTtcXg5wQ4VSqcYlUCx58UOSuWvmOPMHiges,2985
+itn/chinese/rules/money.py,sha256=lNixki_73PNefQUECRUYebmD461klNXTtKQm61o4Nu8,2160
+itn/chinese/rules/postprocessor.py,sha256=Eq9aZc7T5OkZGQB15r2qeT8SblS5cvnEosqHjLTN8To,1144
+itn/chinese/rules/time.py,sha256=4sX_3_lhnjQh0fkEBnP7rGnoTPbquHDXaV5lbeGW75M,1931
+itn/chinese/rules/whitelist.py,sha256=UuvIORHZiezfAKWK7qL3QByTGxHMXVnOZp-lYrwmewk,1081
+itn/chinese/test/__init__.py,sha256=eOXRztjSZS0xmUruH2eKjdHx6RtmMxBtae5aFxKrgaE,605
+itn/chinese/test/normalizer_test.py,sha256=a9nqt-is07zYOl3XDAhjUyNzBlwBe3qdaHtvTT6dWn4,4832
+itn/chinese/test/utils.py,sha256=aCZH5BL0dghGjemhVwfISBUxLFMG_ceMzwzuOb5ht28,1094
+tn/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+tn/__main__.py,sha256=OUMsgyp8SWb7raJY_py61uM29j023yE5po95O9yinzk,64
+tn/main.py,sha256=Q45id3b2g71MPRLnaHpo8iWPc38MeyZKigNd5FinNvI,3578
+tn/processor.py,sha256=aAcHF3mwyhbPDAwxZKD6tPho-xL6ZNJMrHuw5e7TaQo,4258
+tn/token_parser.py,sha256=nsTYzug-gq2HYnTJHja_HXntoRVX3gITQAlI9z8WvWU,4487
+tn/utils.py,sha256=lJLuW2jkobgXCyNYl29zhtyb3B80CS7Pqz7zqHRVr4c,2792
+tn/zh_tn_tagger.fst,sha256=zZ7zLr-Y774X3lb9bnmUv9Fs3Gv-LXa44X0rMFL6a9g,3477714
+tn/zh_tn_verbalizer.fst,sha256=VoZnjew7YiDRoDC6TgDHKXhrkcSGOz030Pnqad8JhcE,558366
+tn/chinese/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+tn/chinese/normalizer.py,sha256=kCeK1Ab-58RVfpcNsvj_TL9mClsPFjTgjLnOxWFKbAI,3848
+tn/chinese/rules/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+tn/chinese/rules/cardinal.py,sha256=qPX9phKDlOctzKjwrru9kBprzb6rE6wyvj3WmHhID4E,3611
+tn/chinese/rules/char.py,sha256=wWnTfzJqE5j7kvEao9rKSN29OVFGNnL25MxS7PcfWEM,965
+tn/chinese/rules/date.py,sha256=Q1uqPOGSB6UJcrBY3ERs-gj_Caq9yA6Uxmk1NTfzmZo,2382
+tn/chinese/rules/fraction.py,sha256=8K7YQp-QWXJI6iOswMS1PSf6-ocBnLOKiqzv8FIWHfc,1520
+tn/chinese/rules/math.py,sha256=PB62l3LT9Y6LqURfE3driJHrDTmdxtPHSBSxwzpl3t0,1547
+tn/chinese/rules/measure.py,sha256=pjkIzWRi2YxzDlkJoGtqde7vl5foCJ7xjvW70pHZTYs,2802
+tn/chinese/rules/money.py,sha256=Bu9rNGIBP3QvbceyS4_NrHPfcFISf70vtoffiV_Ahuc,1567
+tn/chinese/rules/postprocessor.py,sha256=a3wA2Aqg-MtyT-it_bqdj9_oJ2bG47gfzKwaL7dgXpk,2100
+tn/chinese/rules/preprocessor.py,sha256=5pQ98OlMFxf8V3C74dTQc0uWeKUhZLA3jNVi61Z4fOA,1092
+tn/chinese/rules/sport.py,sha256=6NSWEiJhSX3aymY5263JDgZkdUZP8BYdJFAc8l51R1c,1726
+tn/chinese/rules/time.py,sha256=mfdAb7whCc0li6cDQKIwi10xcaM7EIjfI8mK7lIFyYQ,2047
+tn/chinese/rules/whitelist.py,sha256=JzrL-UfedP7FSps5s4xhSjfEcCkVKzEEGdkZfE96i2E,1652
+tn/chinese/test/__init__.py,sha256=eOXRztjSZS0xmUruH2eKjdHx6RtmMxBtae5aFxKrgaE,605
+tn/chinese/test/cardinal_test.py,sha256=o0LVTOMNMIgwADl7wYHRBBH6s2fLli9kU5BaYxdIHBc,1246
+tn/chinese/test/char_test.py,sha256=p4vG0iwc2FYn0uTbQQzWBq-TLgfIWMfgKqfu1vpP50o,954
+tn/chinese/test/date_test.py,sha256=TEF8wYGl1Ubr_5daSnp92cJY2yCPv81aZ9j3BfA8osU,954
+tn/chinese/test/fraction_test.py,sha256=w2Pa7yiHG85TbU8XWf-ZDbB49g52bI1pDOEZUdMxVnE,994
+tn/chinese/test/math_test.py,sha256=w__bpbDFK4VsGK-fWenHfV9Y_SImBb6q9g-twplbqGc,954
+tn/chinese/test/measure_test.py,sha256=k76Dz32DXvl6bBjbceISH4Xiz6xAhBwJRv3OYCvp0OE,982
+tn/chinese/test/money_test.py,sha256=MrQQTJQHPql1mNIOljomlLyqIm-MS5xlvyIGT7KaboU,964
+tn/chinese/test/normalizer_test.py,sha256=rRf9xpGpnhAICwnAhcMqOg5RSvJUxvSc5qy2U5kM1ak,1597
+tn/chinese/test/postprocessor_test.py,sha256=KNXdVJQ3AyCsPqjiiFxk9QQoALUDRcSDvzsLa2OGxd4,1099
+tn/chinese/test/preprocessor_test.py,sha256=ipNvTn0MYcR0MDyOQuT_hR8DKiDHWoYZb4coANX31Kk,1082
+tn/chinese/test/sport_test.py,sha256=ZF4QpgbfIZ38bwnCE5a9gkR1jv1N6RvN4VECHQImOl8,964
+tn/chinese/test/time_test.py,sha256=hLi3gCwVrV4oOdStWCwOeT6c9XTOatjOB8pFfbe6r0I,953
+tn/chinese/test/token_parser_test.py,sha256=8JcSqlvaYTZ46rZgMsfAVNmcYez6t0gonhnx4nc-1ew,2614
+tn/chinese/test/utils.py,sha256=aCZH5BL0dghGjemhVwfISBUxLFMG_ceMzwzuOb5ht28,1094
+tn/chinese/test/whitelist_test.py,sha256=oPB-7_g8l6zhAeH5fp7NlaRJYcDNz7VAOKwhCnebGdc,1003
+tn/english/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+tn/english/normalizer.py,sha256=vC5GwQkHpiSSwF-ASzBBWfk3t4Xccslkez4yP93Mp90,3777
+tn/english/data/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+tn/english/data/address/__init__.py,sha256=EqFAHHjK7kJqR_FV8vBLddBJWhbVD8pREEfm8C5nSEE,623
+tn/english/data/date/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+tn/english/data/electronic/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+tn/english/data/measure/__init__.py,sha256=TS5NMVLZXlTIgbt7grYmlxBfRKrWELPnT9DedX9kYBk,610
+tn/english/data/money/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+tn/english/data/number/__init__.py,sha256=TS5NMVLZXlTIgbt7grYmlxBfRKrWELPnT9DedX9kYBk,610
+tn/english/data/ordinal/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+tn/english/data/roman/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+tn/english/data/telephone/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+tn/english/data/time/__init__.py,sha256=TS5NMVLZXlTIgbt7grYmlxBfRKrWELPnT9DedX9kYBk,610
+tn/english/data/whitelist/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+tn/english/rules/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+tn/english/rules/cardinal.py,sha256=w3HvKbq86mZjb3MhXc96D_Ajaem1raCHs0pCGvhQaRs,8424
+tn/english/rules/date.py,sha256=QoUczkPwAeYanT41_8Qpi_FqvuSfvXX5LHdEgcsLkVA,13683
+tn/english/rules/decimal.py,sha256=s49eTpngiE4BZyLvfgsg45kpcemElWTLGx7VuxVR1Rw,8795
+tn/english/rules/electronic.py,sha256=2XBnz3usFyFame2WcInVrtZEW5TPXXowANpLeBEjD7I,9190
+tn/english/rules/fraction.py,sha256=ltGhbd1Qeb-BsuVCPYq8cc__k0Xkkn4WBUiz6dFs8J8,5685
+tn/english/rules/measure.py,sha256=DjQnq_9UXL-pHdqaHkp3oUv4qEhiGB_764YmUEq9v6w,14025
+tn/english/rules/money.py,sha256=0hjQWUdI4wWsOVKkkBs5lvFsQaMIWOQcs6oOLBhQzDM,5457
+tn/english/rules/ordinal.py,sha256=xFUaNvPx2rKDJz3TMiE2RKje6gcvcVImr3sXixWfOKU,3989
+tn/english/rules/punctuation.py,sha256=KyypaQs7SsQC1RCv-h4NGu4Jm-PvuSGUyM3mtPbCMws,2979
+tn/english/rules/roman.py,sha256=pQFuSIjzWnMN-jiSuVqv3AxJEA9kA8s0NUgGpCCVbiU,6051
+tn/english/rules/telephone.py,sha256=JkhHZU1O2h4a_W3QHVBY1lOQvq4Tys5JwAlL1Kkt8Pg,8275
+tn/english/rules/time.py,sha256=7quSQrLifeJ4WLwAyDkuvGe1-lNomN_MjvFAP3zHBas,7764
+tn/english/rules/whitelist.py,sha256=5pIEZyCadKmS7AAJ7G7JQhWFvo2oPszy0WxxIAMZr_E,6097
+tn/english/rules/word.py,sha256=ySAa3qn6Wf1Qze9limjPfYJd37Kk2Juz9AZseGEFNu8,2366
+tn/english/test/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+tn/english/test/cardinal_test.py,sha256=F7vP5ZIOxoQhRJ0Se597bHinRnFU_bOd2PzTo9M0xmg,1013
+tn/english/test/date_test.py,sha256=dFOJXAdU19i3i7KZdt89PQ-c0EeGmjUhQCXT8bFE71o,973
+tn/english/test/decimal_test.py,sha256=28Q4YCEPGRoQ-6s3zroyxUbVZ2GiALxZXjXmQcs9WwQ,1003
+tn/english/test/electronic_test.py,sha256=iWgt4lFHDfrZZLyBAMSL5AOjATeFK1CrUDLLWUMll8I,1033
+tn/english/test/fraction_test.py,sha256=aQrDwqjjfDN0CWM34VOOTINfos0MHzuY6Cjhv7cF9jE,1013
+tn/english/test/measure_test.py,sha256=14dGDfjdbYiH0MYBmPD9JvfVAXW_-Fe4rPCrWJ72T-A,1003
+tn/english/test/money_test.py,sha256=oVhT7JpTaTekOPuMLmaubSv5912ow35OsPquikULi2A,983
+tn/english/test/normalizer_test.py,sha256=wNq2SE2DyGmDQQ4D1yFrYszE8czFvGOlvKwsCX2ymKQ,1006
+tn/english/test/ordinal_test.py,sha256=sqrPIy9ZABNSKQaadU-GfYydnh3dUxk2SfkjlZfHfp4,1003
+tn/english/test/roman_test.py,sha256=Lemo7K4I-EoJ34yeT9VVLTHOSOYVzpK3xnCQ38Fer5w,983
+tn/english/test/telephone_test.py,sha256=I9xnKNXfWagRTHS8hvBvJKAqeFvH8_sjD5G3n8pEwys,1023
+tn/english/test/time_test.py,sha256=4E_PS-D5caTUZgrI-UL7q0aoqmtD8np7yeXUkGyc1Qo,973
+tn/english/test/utils.py,sha256=aCZH5BL0dghGjemhVwfISBUxLFMG_ceMzwzuOb5ht28,1094
+tn/english/test/whitelist_test.py,sha256=fnlpnwiDgQWsl5ONZklWXEyLQoUSuwhdowcV7a8fXvw,1023
+tn/english/test/word_test.py,sha256=ACguloXPl4Ttw8fj9F_wvr1coaQAoj_MLOJ74wyy-WY,954
+WeTextProcessing-0.2.0.dist-info/LICENSE,sha256=xx0jnfkXJvxRnG63LTGOxlggYnIysveWIZ6H3PNdCrQ,11357
+WeTextProcessing-0.2.0.dist-info/METADATA,sha256=xQnfep16ju1AcZJMNvB7AAbq_nKXqUonmz1kOqw29K4,5178
+WeTextProcessing-0.2.0.dist-info/WHEEL,sha256=GJ7t_kWBFywbagK5eo9IoUwLW6oyOeTKmQ-9iHFVNxQ,92
+WeTextProcessing-0.2.0.dist-info/entry_points.txt,sha256=iAd8tFSNA8AY5fUECPlk9KaDFC3ISvDjoKAZVsCWPjE,61
+WeTextProcessing-0.2.0.dist-info/top_level.txt,sha256=6NasE_UheQezX9_0V3MbiOSnTQpvM_funMjv4wpMGgA,7
+WeTextProcessing-0.2.0.dist-info/RECORD,,

{WeTextProcessing-0.1.11.dist-info → WeTextProcessing-0.2.0.dist-info}/WHEEL

@@ -1,5 +1,5 @@
 Wheel-Version: 1.0
-Generator: bdist_wheel (0.42.0)
+Generator: bdist_wheel (0.43.0)
 Root-Is-Purelib: true
 Tag: py3-none-any
 

itn/chinese/rules/cardinal.py

@@ -36,6 +36,10 @@ class Cardinal(Processor):
     def build_tagger(self):
         zero = string_file('itn/chinese/data/number/zero.tsv')  # 0
         digit = string_file('itn/chinese/data/number/digit.tsv')  # 1 ~ 9
+        special_tilde = string_file(
+            'itn/chinese//data/number/special_tilde.tsv')  # 七八十->70~80
+        special_dash = string_file(
+            'itn/chinese//data/number/special_dash.tsv')  # 七八十->70-80
         sign = string_file('itn/chinese/data/number/sign.tsv')  # + -
         dot = string_file('itn/chinese/data/number/dot.tsv')  # .
 
@@ -82,7 +86,7 @@ class Cardinal(Processor):
             ten_thousand |= (thousand | hundred) + accep("万") + delete(
                 "零").ques + (thousand | hundred | tens | teen | digits).ques
 
-        # 1. 利用基础数字所构建的包含0~9的完整数字
+        # 1. 利用基础数字所构建的包含0~9的标准数字
         # 个/十/百/千/万
         number = digits | teen | tens | hundred | thousand | ten_thousand
         # 兆/亿
@@ -90,25 +94,19 @@ class Cardinal(Processor):
                   (number + accep('亿') + delete('零').ques).ques + number)
         # 负的xxx 1.11, 1.01
         number = sign.ques + number + (dot + digits.plus).ques
-        # 五六万，三五千，六七百，三四十
-        special_2number = digit + insert("0~") + digit + cross("十", "0")
-        special_2number |= digit + insert("00~") + digit + cross("百", "00")
-        special_2number |= digit + insert("000~") + digit + cross("千", "000")
-        special_2number |= digit + insert("0000~") + digit + cross("万", "0000")
-        number |= special_2number
-        # 十七八美元 => $17~18, 四十五六岁 => 45-6岁,
-        # 三百七八公里 => 370-80km, 三百七八十千克 => 370-80kg
-        special_3number = cross('十', '1') + digit + insert("~1") + digit
-        special_3number |= digit + delete('十') + digit + insert("-") + digit
-        special_3number |= digit + delete('百') + digit + insert("0-") + digit \
-            + (insert("0") | add_weight(cross("十", "0"), -0.1))
-        number |= add_weight(special_3number, -100.0)
+        # 五六万 => 5~6万，三五千 => 3000~5000，六七百 => 600~700，三四十 => 30~40
+        number |= special_tilde
+        # 十七八 => 17-8, 四十五六 => 45-6, 三百七八十 => 370-80
+        _special_dash = cross('十', '1') + special_dash
+        _special_dash |= digit + delete('十') + special_dash
+        _special_dash |= digit + delete('百') + special_dash
+        number |= _special_dash
 
         self.number = number.optimize()
-        self.special_2number = special_2number.optimize()
-        self.special_3number = special_3number.optimize()
+        self.special_tilde = special_tilde.optimize()
+        self.special_dash = _special_dash.optimize()
 
-        # 2. 利用基础数字所构建的不包含0~9的完整数字
+        # 2. 利用基础数字所构建的不包含0~9的标准数字
         # 十/百/千/万
         number_exclude_0_to_9 = teen | tens | hundred | thousand | ten_thousand
         # 兆/亿
@@ -123,8 +121,8 @@ class Cardinal(Processor):
         # 五六万，三五千，六七百，三四十
         # 十七八美元 => $17~18, 四十五六岁 => 45-6岁,
         # 三百七八公里 => 370-80km, 三百七八十千克 => 370-80kg
-        number_exclude_0_to_9 |= special_2number
-        number_exclude_0_to_9 |= add_weight(special_3number, -100.0)
+        number_exclude_0_to_9 |= special_tilde
+        number_exclude_0_to_9 |= add_weight(_special_dash, -100.0)
 
         self.number_exclude_0_to_9 = (sign.ques +
                                       number_exclude_0_to_9).optimize()
@@ -136,16 +134,19 @@ class Cardinal(Processor):
         cardinal |= (number + dot + digits.plus)
         # cardinal string like 110 or 12306 or 13125617878, used in phone,
         #   340621199806051223, used in ID card
+        idcard_last_char = digits | 'X' | 'x'
         cardinal |= (digits**3 | digits**4 | digits**5 | digits**11
-                     | digits**18)
+                     | (digits**17 + idcard_last_char) | digits**18)
 
-        # 4. 特殊格式的数字 + 包含或不包含0~9的完整数字
+        # 4. 特殊格式的数字 + 标准数字
         # cardinal string like 23
         if self.enable_standalone_number:
             if self.enable_0_to_9:
-                cardinal |= number
+                # 特殊格式数字为第一优先级, 标准数字为第二优先级, 如 "一二三四"
+                # 优先转译为 "1234" 而非 "1~2 3~4"
+                cardinal |= add_weight(number, 0.1)
             else:
-                cardinal |= number_exclude_0_to_9
+                cardinal |= add_weight(number_exclude_0_to_9, 0.1)
         tagger = insert('value: "') + cardinal + (insert(" ") + cardinal).star \
             + insert('"')
         self.tagger = self.add_tokens(tagger)

itn/chinese/rules/license_plate.py

@@ -27,8 +27,11 @@ class LicensePlate(Processor):
 
     def build_tagger(self):
         digit = string_file('itn/chinese/data/number/digit.tsv')  # 1 ~ 9
+        zero = string_file('itn/chinese/data/number/zero.tsv')  # 0
+        digits = zero | digit
         province = string_file(
             'itn/chinese/data/license_plate/province.tsv')  # 皖
-        license_plate = province + self.ALPHA + (self.ALPHA | digit)**5
+        license_plate = province + self.ALPHA + (self.ALPHA | digits)**5
+        license_plate |= province + self.ALPHA + (self.ALPHA | digits)**6
         tagger = insert('value: "') + license_plate + insert('"')
         self.tagger = self.add_tokens(tagger)

itn/chinese/rules/measure.py

@@ -31,7 +31,6 @@ class Measure(Processor):
     def build_tagger(self):
         units_en = string_file('itn/chinese/data/measure/units_en.tsv')
         units_zh = string_file('itn/chinese/data/measure/units_zh.tsv')
-        digit = string_file('itn/chinese/data/number/digit.tsv')  # 1 ~ 9
         sign = string_file('itn/chinese/data/number/sign.tsv')  # + -
         to = cross('到', '~') | cross('到百分之', '~')
 
@@ -52,8 +51,6 @@ class Measure(Processor):
 
         # 十千米每小时 => 10km/h, 十一到一百千米每小时 => 11~100km/h
         measure = number + (to + number).ques + units
-        # 七八块钱
-        measure |= digit + insert("~") + digit + units
         tagger = insert('value: "') + (measure | percent) + insert('"')
 
         # 每小时十千米 => 10km/h, 每小时三十到三百一十一千米 => 30~311km/h

tn/chinese/rules/cardinal.py

@@ -64,11 +64,13 @@ class Cardinal(Processor):
         number @= self.build_rule(
             cross('二百', '两百')
             | cross('二千', '两千')
-            | cross('二万', '两万'))
-        self.number = accep('约').ques + accep('人均').ques + number.optimize()
+            | cross('二万', '两万')).optimize()
+        percent = insert('百分之') + number + delete('%')
+        self.number = accep('约').ques + accep('人均').ques + (number | percent)
 
         # cardinal string like 127.0.0.1, used in ID, IP, etc.
         cardinal = digits.plus + (dot + digits.plus)**3
+        cardinal |= percent
         # xxxx-xxx-xxx
         cardinal |= digits.plus + (delete('-') + digits.plus)**2
         # xxx-xxxxxxxx
@@ -76,6 +78,7 @@ class Cardinal(Processor):
         # three or five or eleven phone numbers
         phone_digits = digits @ self.build_rule(cross('一', '幺'))
         phone = phone_digits**3 | phone_digits**5 | phone_digits**11
+        phone |= accep("尾号") + (accep("是") | accep("为")).ques + phone_digits**4
         cardinal |= add_weight(phone, -1.0)
 
         tagger = insert('value: "') + cardinal + insert('"')

tn/chinese/rules/math.py

@@ -15,23 +15,32 @@
 from tn.chinese.rules.cardinal import Cardinal
 from tn.processor import Processor
 
-from pynini import string_file
+from pynini import cross, string_file
 from pynini.lib.pynutil import delete, insert
 
 
 class Math(Processor):
 
     def __init__(self):
-        super().__init__(name='math')
+        super().__init__(name="math")
         self.build_tagger()
         self.build_verbalizer()
 
     def build_tagger(self):
-        operator = string_file('tn/chinese/data/math/operator.tsv')
+        operator = string_file("tn/chinese/data/math/operator.tsv")
+        # When it appears alone, it is treated as punctuation
+        symbols = (
+            cross("~", "到")
+            | cross(":", "比")
+            | cross("<", "小于")
+            | cross(">", "大于")
+        )
 
         number = Cardinal().number
-        tagger = (number + ((delete(' ').ques + operator + delete(' ').ques +
-                             number).plus).ques)
+        tagger = (
+            number
+            + (delete(" ").ques + (operator | symbols) + delete(" ").ques + number).star
+        )
         tagger |= operator
         tagger = insert('value: "') + tagger + insert('"')
         self.tagger = self.add_tokens(tagger)

tn/chinese/rules/measure.py

@@ -35,8 +35,6 @@ class Measure(Processor):
         to = cross('-', '到') | cross('~', '到') | accep('到')
 
         number = Cardinal().number
-        percent = insert('百分之') + number + delete('%')
-
         number @= self.build_rule(cross('二', '两'), '[BOS]', '[EOS]')
         # 1-11个，1个-11个
         prefix = number + (rmspace + units).ques + to
@@ -55,8 +53,7 @@ class Measure(Processor):
         prefix = cardinal + (rmspace + unit).ques + to
         annual = prefix.ques + cardinal + unit
 
-        tagger = insert('value: "') + (measure | percent
-                                       | annual) + insert('"')
+        tagger = insert('value: "') + (measure | annual) + insert('"')
 
         # 10km/h
         rmsign = rmspace + delete('/') + rmspace

tn/english/data/address/__init__.py

@@ -0,0 +1,13 @@
+# Copyright (c) 2021, NVIDIA CORPORATION & AFFILIATES.  All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.

tn/english/data/measure/__init__.py

@@ -0,0 +1,13 @@
+# Copyright (c) 2021, NVIDIA CORPORATION.  All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.

tn/english/data/number/__init__.py

@@ -0,0 +1,13 @@
+# Copyright (c) 2021, NVIDIA CORPORATION.  All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.

tn/english/data/time/__init__.py

@@ -0,0 +1,13 @@
+# Copyright (c) 2021, NVIDIA CORPORATION.  All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.

tn/english/normalizer.py

@@ -0,0 +1,92 @@
+# Copyright (c) 2022 Zhendong Peng (pzd17@tsinghua.org.cn)
+# Copyright (c) 2024 Xingchen Song (sxc19@tsinghua.org.cn)
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+from tn.processor import Processor
+from tn.english.rules.cardinal import Cardinal
+from tn.english.rules.ordinal import Ordinal
+from tn.english.rules.decimal import Decimal
+from tn.english.rules.fraction import Fraction
+from tn.english.rules.word import Word
+from tn.english.rules.date import Date
+from tn.english.rules.time import Time
+from tn.english.rules.measure import Measure
+from tn.english.rules.money import Money
+from tn.english.rules.telephone import Telephone
+from tn.english.rules.electronic import Electronic
+from tn.english.rules.whitelist import WhiteList
+from tn.english.rules.punctuation import Punctuation
+
+from pynini.lib.pynutil import add_weight, delete
+from importlib_resources import files
+
+
+class Normalizer(Processor):
+
+    def __init__(self, cache_dir=None, overwrite_cache=False):
+        super().__init__(name='en_normalizer', ordertype="en_tn")
+        if cache_dir is None:
+            cache_dir = files("tn")
+        self.build_fst('en_tn', cache_dir, overwrite_cache)
+
+    def build_tagger(self):
+        cardinal = add_weight(Cardinal().tagger, 1.0)
+        ordinal = add_weight(Ordinal().tagger, 1.0)
+        decimal = add_weight(Decimal().tagger, 1.0)
+        fraction = add_weight(Fraction().tagger, 1.0)
+        date = add_weight(Date().tagger, 0.99)
+        time = add_weight(Time().tagger, 1.00)
+        measure = add_weight(Measure().tagger, 1.00)
+        money = add_weight(Money().tagger, 1.00)
+        telephone = add_weight(Telephone().tagger, 1.00)
+        electronic = add_weight(Electronic().tagger, 1.00)
+        word = add_weight(Word().tagger, 100)
+        whitelist = add_weight(WhiteList().tagger, 1.00)
+        punct = add_weight(Punctuation().tagger, 2.00)
+        # TODO(xcsong): add roman
+        tagger = punct.star + \
+            (cardinal | ordinal | word
+             | date | decimal | fraction
+             | time | measure | money
+             | telephone | electronic
+             | whitelist
+             | punct).optimize() + (punct.plus | self.DELETE_SPACE)
+        # delete the last space
+        self.tagger = tagger.star @ self.build_rule(delete(' '), r='[EOS]')
+
+    def build_verbalizer(self):
+        cardinal = Cardinal().verbalizer
+        ordinal = Ordinal().verbalizer
+        decimal = Decimal().verbalizer
+        fraction = Fraction().verbalizer
+        word = Word().verbalizer
+        date = Date().verbalizer
+        time = Time().verbalizer
+        measure = Measure().verbalizer
+        money = Money().verbalizer
+        telephone = Telephone().verbalizer
+        electronic = Electronic().verbalizer
+        whitelist = WhiteList().verbalizer
+        punct = Punctuation().verbalizer
+        verbalizer = \
+            (cardinal | ordinal | word
+             | date | decimal
+             | fraction | time
+             | measure | money
+             | telephone
+             | electronic
+             | whitelist
+             | punct).optimize() + punct.ques + self.INSERT_SPACE
+        self.verbalizer = verbalizer.star @ self.build_rule(delete(' '),
+                                                            r='[EOS]')