WeTextProcessing 0.1.12__py3-none-any.whl → 0.2.0__py3-none-any.whl

{WeTextProcessing-0.1.12.dist-info → WeTextProcessing-0.2.0.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: WeTextProcessing
-Version: 0.1.12
+Version: 0.2.0
 Summary: WeTextProcessing, including TN & ITN
 Home-page: https://github.com/wenet-e2e/WeTextProcessing
 Author: Zhendong Peng, Xingchen Song

WeTextProcessing-0.2.0.dist-info/RECORD

@@ -0,0 +1,112 @@
+itn/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+itn/__main__.py,sha256=jFh8bEB_lC5y7ab9VMqAxWRaI8D-SpBSTiHfMJC1b10,65
+itn/main.py,sha256=NnT9Sh3VTEI0sX_DO9IGVsXCWtZyfRbzj9Sj_6cBxqM,2610
+itn/zh_itn_tagger.fst,sha256=vRgLHTqqchvubK6kKt8_fHhbxRfBykCbLQfhFMk_I64,1080610
+itn/zh_itn_verbalizer.fst,sha256=pHXRUfzls2UmWwTmHHhYmBWMusIfWpU3I7IwxeRjn28,119018
+itn/chinese/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+itn/chinese/inverse_normalizer.py,sha256=HCNW5WCHEYeu60f8lxRmvpXh_48GcXa036EFFR9PQo0,3560
+itn/chinese/rules/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+itn/chinese/rules/cardinal.py,sha256=x3pBfnqzWr85BMkYSOJ87IPL_vAqrCGrUDJvugbq5ZY,7385
+itn/chinese/rules/char.py,sha256=wWnTfzJqE5j7kvEao9rKSN29OVFGNnL25MxS7PcfWEM,965
+itn/chinese/rules/date.py,sha256=dA_AQTXaKfO9U8Mtp-ZT9bYdsaOhp5GHC1UpArNm5DY,2319
+itn/chinese/rules/fraction.py,sha256=cYfBcV3XrXo7SIKFOfvOKjFITnhqxJkyiEaU8byiI8A,1966
+itn/chinese/rules/license_plate.py,sha256=DX9JJjQhh8L6G_8fkVWuabABtw5wjFwQdfS3zxili6M,1439
+itn/chinese/rules/math.py,sha256=WmcZQdns7pY4v5gQajEacnHbedP3w_KJVn3WUvoeZkE,1199
+itn/chinese/rules/measure.py,sha256=JJSdhporTtcXg5wQ4VSqcYlUCx58UOSuWvmOPMHiges,2985
+itn/chinese/rules/money.py,sha256=lNixki_73PNefQUECRUYebmD461klNXTtKQm61o4Nu8,2160
+itn/chinese/rules/postprocessor.py,sha256=Eq9aZc7T5OkZGQB15r2qeT8SblS5cvnEosqHjLTN8To,1144
+itn/chinese/rules/time.py,sha256=4sX_3_lhnjQh0fkEBnP7rGnoTPbquHDXaV5lbeGW75M,1931
+itn/chinese/rules/whitelist.py,sha256=UuvIORHZiezfAKWK7qL3QByTGxHMXVnOZp-lYrwmewk,1081
+itn/chinese/test/__init__.py,sha256=eOXRztjSZS0xmUruH2eKjdHx6RtmMxBtae5aFxKrgaE,605
+itn/chinese/test/normalizer_test.py,sha256=a9nqt-is07zYOl3XDAhjUyNzBlwBe3qdaHtvTT6dWn4,4832
+itn/chinese/test/utils.py,sha256=aCZH5BL0dghGjemhVwfISBUxLFMG_ceMzwzuOb5ht28,1094
+tn/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+tn/__main__.py,sha256=OUMsgyp8SWb7raJY_py61uM29j023yE5po95O9yinzk,64
+tn/main.py,sha256=Q45id3b2g71MPRLnaHpo8iWPc38MeyZKigNd5FinNvI,3578
+tn/processor.py,sha256=aAcHF3mwyhbPDAwxZKD6tPho-xL6ZNJMrHuw5e7TaQo,4258
+tn/token_parser.py,sha256=nsTYzug-gq2HYnTJHja_HXntoRVX3gITQAlI9z8WvWU,4487
+tn/utils.py,sha256=lJLuW2jkobgXCyNYl29zhtyb3B80CS7Pqz7zqHRVr4c,2792
+tn/zh_tn_tagger.fst,sha256=zZ7zLr-Y774X3lb9bnmUv9Fs3Gv-LXa44X0rMFL6a9g,3477714
+tn/zh_tn_verbalizer.fst,sha256=VoZnjew7YiDRoDC6TgDHKXhrkcSGOz030Pnqad8JhcE,558366
+tn/chinese/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+tn/chinese/normalizer.py,sha256=kCeK1Ab-58RVfpcNsvj_TL9mClsPFjTgjLnOxWFKbAI,3848
+tn/chinese/rules/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+tn/chinese/rules/cardinal.py,sha256=qPX9phKDlOctzKjwrru9kBprzb6rE6wyvj3WmHhID4E,3611
+tn/chinese/rules/char.py,sha256=wWnTfzJqE5j7kvEao9rKSN29OVFGNnL25MxS7PcfWEM,965
+tn/chinese/rules/date.py,sha256=Q1uqPOGSB6UJcrBY3ERs-gj_Caq9yA6Uxmk1NTfzmZo,2382
+tn/chinese/rules/fraction.py,sha256=8K7YQp-QWXJI6iOswMS1PSf6-ocBnLOKiqzv8FIWHfc,1520
+tn/chinese/rules/math.py,sha256=PB62l3LT9Y6LqURfE3driJHrDTmdxtPHSBSxwzpl3t0,1547
+tn/chinese/rules/measure.py,sha256=pjkIzWRi2YxzDlkJoGtqde7vl5foCJ7xjvW70pHZTYs,2802
+tn/chinese/rules/money.py,sha256=Bu9rNGIBP3QvbceyS4_NrHPfcFISf70vtoffiV_Ahuc,1567
+tn/chinese/rules/postprocessor.py,sha256=a3wA2Aqg-MtyT-it_bqdj9_oJ2bG47gfzKwaL7dgXpk,2100
+tn/chinese/rules/preprocessor.py,sha256=5pQ98OlMFxf8V3C74dTQc0uWeKUhZLA3jNVi61Z4fOA,1092
+tn/chinese/rules/sport.py,sha256=6NSWEiJhSX3aymY5263JDgZkdUZP8BYdJFAc8l51R1c,1726
+tn/chinese/rules/time.py,sha256=mfdAb7whCc0li6cDQKIwi10xcaM7EIjfI8mK7lIFyYQ,2047
+tn/chinese/rules/whitelist.py,sha256=JzrL-UfedP7FSps5s4xhSjfEcCkVKzEEGdkZfE96i2E,1652
+tn/chinese/test/__init__.py,sha256=eOXRztjSZS0xmUruH2eKjdHx6RtmMxBtae5aFxKrgaE,605
+tn/chinese/test/cardinal_test.py,sha256=o0LVTOMNMIgwADl7wYHRBBH6s2fLli9kU5BaYxdIHBc,1246
+tn/chinese/test/char_test.py,sha256=p4vG0iwc2FYn0uTbQQzWBq-TLgfIWMfgKqfu1vpP50o,954
+tn/chinese/test/date_test.py,sha256=TEF8wYGl1Ubr_5daSnp92cJY2yCPv81aZ9j3BfA8osU,954
+tn/chinese/test/fraction_test.py,sha256=w2Pa7yiHG85TbU8XWf-ZDbB49g52bI1pDOEZUdMxVnE,994
+tn/chinese/test/math_test.py,sha256=w__bpbDFK4VsGK-fWenHfV9Y_SImBb6q9g-twplbqGc,954
+tn/chinese/test/measure_test.py,sha256=k76Dz32DXvl6bBjbceISH4Xiz6xAhBwJRv3OYCvp0OE,982
+tn/chinese/test/money_test.py,sha256=MrQQTJQHPql1mNIOljomlLyqIm-MS5xlvyIGT7KaboU,964
+tn/chinese/test/normalizer_test.py,sha256=rRf9xpGpnhAICwnAhcMqOg5RSvJUxvSc5qy2U5kM1ak,1597
+tn/chinese/test/postprocessor_test.py,sha256=KNXdVJQ3AyCsPqjiiFxk9QQoALUDRcSDvzsLa2OGxd4,1099
+tn/chinese/test/preprocessor_test.py,sha256=ipNvTn0MYcR0MDyOQuT_hR8DKiDHWoYZb4coANX31Kk,1082
+tn/chinese/test/sport_test.py,sha256=ZF4QpgbfIZ38bwnCE5a9gkR1jv1N6RvN4VECHQImOl8,964
+tn/chinese/test/time_test.py,sha256=hLi3gCwVrV4oOdStWCwOeT6c9XTOatjOB8pFfbe6r0I,953
+tn/chinese/test/token_parser_test.py,sha256=8JcSqlvaYTZ46rZgMsfAVNmcYez6t0gonhnx4nc-1ew,2614
+tn/chinese/test/utils.py,sha256=aCZH5BL0dghGjemhVwfISBUxLFMG_ceMzwzuOb5ht28,1094
+tn/chinese/test/whitelist_test.py,sha256=oPB-7_g8l6zhAeH5fp7NlaRJYcDNz7VAOKwhCnebGdc,1003
+tn/english/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+tn/english/normalizer.py,sha256=vC5GwQkHpiSSwF-ASzBBWfk3t4Xccslkez4yP93Mp90,3777
+tn/english/data/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+tn/english/data/address/__init__.py,sha256=EqFAHHjK7kJqR_FV8vBLddBJWhbVD8pREEfm8C5nSEE,623
+tn/english/data/date/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+tn/english/data/electronic/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+tn/english/data/measure/__init__.py,sha256=TS5NMVLZXlTIgbt7grYmlxBfRKrWELPnT9DedX9kYBk,610
+tn/english/data/money/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+tn/english/data/number/__init__.py,sha256=TS5NMVLZXlTIgbt7grYmlxBfRKrWELPnT9DedX9kYBk,610
+tn/english/data/ordinal/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+tn/english/data/roman/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+tn/english/data/telephone/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+tn/english/data/time/__init__.py,sha256=TS5NMVLZXlTIgbt7grYmlxBfRKrWELPnT9DedX9kYBk,610
+tn/english/data/whitelist/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+tn/english/rules/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+tn/english/rules/cardinal.py,sha256=w3HvKbq86mZjb3MhXc96D_Ajaem1raCHs0pCGvhQaRs,8424
+tn/english/rules/date.py,sha256=QoUczkPwAeYanT41_8Qpi_FqvuSfvXX5LHdEgcsLkVA,13683
+tn/english/rules/decimal.py,sha256=s49eTpngiE4BZyLvfgsg45kpcemElWTLGx7VuxVR1Rw,8795
+tn/english/rules/electronic.py,sha256=2XBnz3usFyFame2WcInVrtZEW5TPXXowANpLeBEjD7I,9190
+tn/english/rules/fraction.py,sha256=ltGhbd1Qeb-BsuVCPYq8cc__k0Xkkn4WBUiz6dFs8J8,5685
+tn/english/rules/measure.py,sha256=DjQnq_9UXL-pHdqaHkp3oUv4qEhiGB_764YmUEq9v6w,14025
+tn/english/rules/money.py,sha256=0hjQWUdI4wWsOVKkkBs5lvFsQaMIWOQcs6oOLBhQzDM,5457
+tn/english/rules/ordinal.py,sha256=xFUaNvPx2rKDJz3TMiE2RKje6gcvcVImr3sXixWfOKU,3989
+tn/english/rules/punctuation.py,sha256=KyypaQs7SsQC1RCv-h4NGu4Jm-PvuSGUyM3mtPbCMws,2979
+tn/english/rules/roman.py,sha256=pQFuSIjzWnMN-jiSuVqv3AxJEA9kA8s0NUgGpCCVbiU,6051
+tn/english/rules/telephone.py,sha256=JkhHZU1O2h4a_W3QHVBY1lOQvq4Tys5JwAlL1Kkt8Pg,8275
+tn/english/rules/time.py,sha256=7quSQrLifeJ4WLwAyDkuvGe1-lNomN_MjvFAP3zHBas,7764
+tn/english/rules/whitelist.py,sha256=5pIEZyCadKmS7AAJ7G7JQhWFvo2oPszy0WxxIAMZr_E,6097
+tn/english/rules/word.py,sha256=ySAa3qn6Wf1Qze9limjPfYJd37Kk2Juz9AZseGEFNu8,2366
+tn/english/test/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+tn/english/test/cardinal_test.py,sha256=F7vP5ZIOxoQhRJ0Se597bHinRnFU_bOd2PzTo9M0xmg,1013
+tn/english/test/date_test.py,sha256=dFOJXAdU19i3i7KZdt89PQ-c0EeGmjUhQCXT8bFE71o,973
+tn/english/test/decimal_test.py,sha256=28Q4YCEPGRoQ-6s3zroyxUbVZ2GiALxZXjXmQcs9WwQ,1003
+tn/english/test/electronic_test.py,sha256=iWgt4lFHDfrZZLyBAMSL5AOjATeFK1CrUDLLWUMll8I,1033
+tn/english/test/fraction_test.py,sha256=aQrDwqjjfDN0CWM34VOOTINfos0MHzuY6Cjhv7cF9jE,1013
+tn/english/test/measure_test.py,sha256=14dGDfjdbYiH0MYBmPD9JvfVAXW_-Fe4rPCrWJ72T-A,1003
+tn/english/test/money_test.py,sha256=oVhT7JpTaTekOPuMLmaubSv5912ow35OsPquikULi2A,983
+tn/english/test/normalizer_test.py,sha256=wNq2SE2DyGmDQQ4D1yFrYszE8czFvGOlvKwsCX2ymKQ,1006
+tn/english/test/ordinal_test.py,sha256=sqrPIy9ZABNSKQaadU-GfYydnh3dUxk2SfkjlZfHfp4,1003
+tn/english/test/roman_test.py,sha256=Lemo7K4I-EoJ34yeT9VVLTHOSOYVzpK3xnCQ38Fer5w,983
+tn/english/test/telephone_test.py,sha256=I9xnKNXfWagRTHS8hvBvJKAqeFvH8_sjD5G3n8pEwys,1023
+tn/english/test/time_test.py,sha256=4E_PS-D5caTUZgrI-UL7q0aoqmtD8np7yeXUkGyc1Qo,973
+tn/english/test/utils.py,sha256=aCZH5BL0dghGjemhVwfISBUxLFMG_ceMzwzuOb5ht28,1094
+tn/english/test/whitelist_test.py,sha256=fnlpnwiDgQWsl5ONZklWXEyLQoUSuwhdowcV7a8fXvw,1023
+tn/english/test/word_test.py,sha256=ACguloXPl4Ttw8fj9F_wvr1coaQAoj_MLOJ74wyy-WY,954
+WeTextProcessing-0.2.0.dist-info/LICENSE,sha256=xx0jnfkXJvxRnG63LTGOxlggYnIysveWIZ6H3PNdCrQ,11357
+WeTextProcessing-0.2.0.dist-info/METADATA,sha256=xQnfep16ju1AcZJMNvB7AAbq_nKXqUonmz1kOqw29K4,5178
+WeTextProcessing-0.2.0.dist-info/WHEEL,sha256=GJ7t_kWBFywbagK5eo9IoUwLW6oyOeTKmQ-9iHFVNxQ,92
+WeTextProcessing-0.2.0.dist-info/entry_points.txt,sha256=iAd8tFSNA8AY5fUECPlk9KaDFC3ISvDjoKAZVsCWPjE,61
+WeTextProcessing-0.2.0.dist-info/top_level.txt,sha256=6NasE_UheQezX9_0V3MbiOSnTQpvM_funMjv4wpMGgA,7
+WeTextProcessing-0.2.0.dist-info/RECORD,,

{WeTextProcessing-0.1.12.dist-info → WeTextProcessing-0.2.0.dist-info}/WHEEL

@@ -1,5 +1,5 @@
 Wheel-Version: 1.0
-Generator: bdist_wheel (0.42.0)
+Generator: bdist_wheel (0.43.0)
 Root-Is-Purelib: true
 Tag: py3-none-any
 

itn/chinese/rules/cardinal.py

@@ -134,8 +134,9 @@ class Cardinal(Processor):
         cardinal |= (number + dot + digits.plus)
         # cardinal string like 110 or 12306 or 13125617878, used in phone,
         #   340621199806051223, used in ID card
+        idcard_last_char = digits | 'X' | 'x'
         cardinal |= (digits**3 | digits**4 | digits**5 | digits**11
-                     | digits**18)
+                     | (digits**17 + idcard_last_char) | digits**18)
 
         # 4. 特殊格式的数字 + 标准数字
         # cardinal string like 23

itn/chinese/rules/license_plate.py

@@ -27,8 +27,11 @@ class LicensePlate(Processor):
 
     def build_tagger(self):
         digit = string_file('itn/chinese/data/number/digit.tsv')  # 1 ~ 9
+        zero = string_file('itn/chinese/data/number/zero.tsv')  # 0
+        digits = zero | digit
         province = string_file(
             'itn/chinese/data/license_plate/province.tsv')  # 皖
-        license_plate = province + self.ALPHA + (self.ALPHA | digit)**5
+        license_plate = province + self.ALPHA + (self.ALPHA | digits)**5
+        license_plate |= province + self.ALPHA + (self.ALPHA | digits)**6
         tagger = insert('value: "') + license_plate + insert('"')
         self.tagger = self.add_tokens(tagger)

tn/chinese/rules/cardinal.py

@@ -64,11 +64,13 @@ class Cardinal(Processor):
         number @= self.build_rule(
             cross('二百', '两百')
             | cross('二千', '两千')
-            | cross('二万', '两万'))
-        self.number = accep('约').ques + accep('人均').ques + number.optimize()
+            | cross('二万', '两万')).optimize()
+        percent = insert('百分之') + number + delete('%')
+        self.number = accep('约').ques + accep('人均').ques + (number | percent)
 
         # cardinal string like 127.0.0.1, used in ID, IP, etc.
         cardinal = digits.plus + (dot + digits.plus)**3
+        cardinal |= percent
         # xxxx-xxx-xxx
         cardinal |= digits.plus + (delete('-') + digits.plus)**2
         # xxx-xxxxxxxx

tn/chinese/rules/math.py

@@ -15,23 +15,32 @@
 from tn.chinese.rules.cardinal import Cardinal
 from tn.processor import Processor
 
-from pynini import string_file
+from pynini import cross, string_file
 from pynini.lib.pynutil import delete, insert
 
 
 class Math(Processor):
 
     def __init__(self):
-        super().__init__(name='math')
+        super().__init__(name="math")
         self.build_tagger()
         self.build_verbalizer()
 
     def build_tagger(self):
-        operator = string_file('tn/chinese/data/math/operator.tsv')
+        operator = string_file("tn/chinese/data/math/operator.tsv")
+        # When it appears alone, it is treated as punctuation
+        symbols = (
+            cross("~", "到")
+            | cross(":", "比")
+            | cross("<", "小于")
+            | cross(">", "大于")
+        )
 
         number = Cardinal().number
-        tagger = (number + ((delete(' ').ques + operator + delete(' ').ques +
-                             number).plus).ques)
+        tagger = (
+            number
+            + (delete(" ").ques + (operator | symbols) + delete(" ").ques + number).star
+        )
         tagger |= operator
         tagger = insert('value: "') + tagger + insert('"')
         self.tagger = self.add_tokens(tagger)

tn/chinese/rules/measure.py

@@ -35,8 +35,6 @@ class Measure(Processor):
         to = cross('-', '到') | cross('~', '到') | accep('到')
 
         number = Cardinal().number
-        percent = insert('百分之') + number + delete('%')
-
         number @= self.build_rule(cross('二', '两'), '[BOS]', '[EOS]')
         # 1-11个，1个-11个
         prefix = number + (rmspace + units).ques + to
@@ -55,8 +53,7 @@ class Measure(Processor):
         prefix = cardinal + (rmspace + unit).ques + to
         annual = prefix.ques + cardinal + unit
 
-        tagger = insert('value: "') + (measure | percent
-                                       | annual) + insert('"')
+        tagger = insert('value: "') + (measure | annual) + insert('"')
 
         # 10km/h
         rmsign = rmspace + delete('/') + rmspace

tn/english/data/address/__init__.py

@@ -0,0 +1,13 @@
+# Copyright (c) 2021, NVIDIA CORPORATION & AFFILIATES.  All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.

tn/english/data/measure/__init__.py

@@ -0,0 +1,13 @@
+# Copyright (c) 2021, NVIDIA CORPORATION.  All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.

tn/english/data/number/__init__.py

@@ -0,0 +1,13 @@
+# Copyright (c) 2021, NVIDIA CORPORATION.  All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.

tn/english/data/time/__init__.py

@@ -0,0 +1,13 @@
+# Copyright (c) 2021, NVIDIA CORPORATION.  All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.

tn/english/normalizer.py

@@ -0,0 +1,92 @@
+# Copyright (c) 2022 Zhendong Peng (pzd17@tsinghua.org.cn)
+# Copyright (c) 2024 Xingchen Song (sxc19@tsinghua.org.cn)
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+from tn.processor import Processor
+from tn.english.rules.cardinal import Cardinal
+from tn.english.rules.ordinal import Ordinal
+from tn.english.rules.decimal import Decimal
+from tn.english.rules.fraction import Fraction
+from tn.english.rules.word import Word
+from tn.english.rules.date import Date
+from tn.english.rules.time import Time
+from tn.english.rules.measure import Measure
+from tn.english.rules.money import Money
+from tn.english.rules.telephone import Telephone
+from tn.english.rules.electronic import Electronic
+from tn.english.rules.whitelist import WhiteList
+from tn.english.rules.punctuation import Punctuation
+
+from pynini.lib.pynutil import add_weight, delete
+from importlib_resources import files
+
+
+class Normalizer(Processor):
+
+    def __init__(self, cache_dir=None, overwrite_cache=False):
+        super().__init__(name='en_normalizer', ordertype="en_tn")
+        if cache_dir is None:
+            cache_dir = files("tn")
+        self.build_fst('en_tn', cache_dir, overwrite_cache)
+
+    def build_tagger(self):
+        cardinal = add_weight(Cardinal().tagger, 1.0)
+        ordinal = add_weight(Ordinal().tagger, 1.0)
+        decimal = add_weight(Decimal().tagger, 1.0)
+        fraction = add_weight(Fraction().tagger, 1.0)
+        date = add_weight(Date().tagger, 0.99)
+        time = add_weight(Time().tagger, 1.00)
+        measure = add_weight(Measure().tagger, 1.00)
+        money = add_weight(Money().tagger, 1.00)
+        telephone = add_weight(Telephone().tagger, 1.00)
+        electronic = add_weight(Electronic().tagger, 1.00)
+        word = add_weight(Word().tagger, 100)
+        whitelist = add_weight(WhiteList().tagger, 1.00)
+        punct = add_weight(Punctuation().tagger, 2.00)
+        # TODO(xcsong): add roman
+        tagger = punct.star + \
+            (cardinal | ordinal | word
+             | date | decimal | fraction
+             | time | measure | money
+             | telephone | electronic
+             | whitelist
+             | punct).optimize() + (punct.plus | self.DELETE_SPACE)
+        # delete the last space
+        self.tagger = tagger.star @ self.build_rule(delete(' '), r='[EOS]')
+
+    def build_verbalizer(self):
+        cardinal = Cardinal().verbalizer
+        ordinal = Ordinal().verbalizer
+        decimal = Decimal().verbalizer
+        fraction = Fraction().verbalizer
+        word = Word().verbalizer
+        date = Date().verbalizer
+        time = Time().verbalizer
+        measure = Measure().verbalizer
+        money = Money().verbalizer
+        telephone = Telephone().verbalizer
+        electronic = Electronic().verbalizer
+        whitelist = WhiteList().verbalizer
+        punct = Punctuation().verbalizer
+        verbalizer = \
+            (cardinal | ordinal | word
+             | date | decimal
+             | fraction | time
+             | measure | money
+             | telephone
+             | electronic
+             | whitelist
+             | punct).optimize() + punct.ques + self.INSERT_SPACE
+        self.verbalizer = verbalizer.star @ self.build_rule(delete(' '),
+                                                            r='[EOS]')

tn/english/rules/cardinal.py

@@ -0,0 +1,186 @@
+# Copyright (c) 2021, NVIDIA CORPORATION.  All rights reserved.
+# Copyright (c) 2024, WENET COMMUNITY.  Xingchen Song (sxc19@tsinghua.org.cn).
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import pynini
+from pynini.examples import plurals
+from pynini.lib import pynutil
+
+from tn.processor import Processor
+from tn.utils import get_abs_path
+
+
+class Cardinal(Processor):
+
+    def __init__(self, deterministic: bool = False):
+        """
+        Args:
+            deterministic: if True will provide a single transduction option,
+                for False multiple transduction are generated (used for audio-based normalization)
+        """
+        super().__init__('cardinal', ordertype="en_tn")
+        self.deterministic = deterministic
+        self.build_tagger()
+        self.build_verbalizer()
+
+    def build_tagger(self):
+        """
+        Finite state transducer for classifying cardinals, e.g.
+            -23 -> cardinal { negative: "true" integer: "twenty three" }
+        """
+        # TODO replace to have "oh" as a default for "0"
+        graph = pynini.Far(
+            get_abs_path(
+                "english/data/number/cardinal_number_name.far")).get_fst()
+        graph_au = pynini.Far(
+            get_abs_path(
+                "english/data/number/cardinal_number_name_au.far")).get_fst()
+        self.graph_hundred_component_at_least_one_none_zero_digit = (
+            pynini.closure(self.DIGIT, 2, 3)
+            | pynini.difference(self.DIGIT, pynini.accep("0"))) @ graph
+
+        graph_digit = pynini.string_file(
+            get_abs_path("english/data/number/digit.tsv"))
+        graph_zero = pynini.string_file(
+            get_abs_path("english/data/number/zero.tsv"))
+
+        single_digits_graph = pynini.invert(graph_digit | graph_zero)
+        self.single_digits_graph = single_digits_graph + pynini.closure(
+            self.INSERT_SPACE + single_digits_graph)
+
+        if not self.deterministic:
+            # for a single token allow only the same normalization
+            # "007" -> {"oh oh seven", "zero zero seven"} not {"oh zero seven"}
+            single_digits_graph_zero = pynini.invert(graph_digit | graph_zero)
+            single_digits_graph_oh = pynini.invert(graph_digit) | pynini.cross(
+                "0", "oh")
+
+            self.single_digits_graph = single_digits_graph_zero + pynini.closure(
+                self.INSERT_SPACE + single_digits_graph_zero)
+            self.single_digits_graph |= single_digits_graph_oh + pynini.closure(
+                self.INSERT_SPACE + single_digits_graph_oh)
+
+            single_digits_graph_with_commas = pynini.closure(
+                self.single_digits_graph + self.INSERT_SPACE, 1,
+                3) + pynini.closure(
+                    pynutil.delete(",") + single_digits_graph +
+                    self.INSERT_SPACE + single_digits_graph +
+                    self.INSERT_SPACE + single_digits_graph, 1)
+
+        graph = (pynini.closure(self.DIGIT, 1, 3) +
+                 (pynini.closure(pynutil.delete(",") + self.DIGIT**3)
+                  | pynini.closure(self.DIGIT**3))) @ graph
+
+        self.graph = graph
+        self.graph_with_and = self.add_optional_and(graph)
+
+        if self.deterministic:
+            long_numbers = pynini.compose(self.DIGIT**(5, ...),
+                                          self.single_digits_graph).optimize()
+            self.long_numbers = plurals._priority_union(
+                long_numbers, self.graph_with_and,
+                pynini.closure(self.VCHAR)).optimize()
+            cardinal_with_leading_zeros = pynini.compose(
+                pynini.accep("0") + pynini.closure(self.DIGIT),
+                self.single_digits_graph)
+            final_graph = self.long_numbers | cardinal_with_leading_zeros
+            final_graph |= self.add_optional_and(graph_au)
+        else:
+            leading_zeros = pynini.compose(
+                pynini.closure(pynini.accep("0"), 1), self.single_digits_graph)
+            cardinal_with_leading_zeros = (
+                leading_zeros + self.INSERT_SPACE + pynini.compose(
+                    pynini.closure(self.DIGIT), self.graph_with_and))
+            self.long_numbers = self.graph_with_and | pynutil.add_weight(
+                self.single_digits_graph, 0.0001)
+            # add small weight to non-default graphs to make sure the deterministic option is listed first
+            final_graph = (self.long_numbers
+                           | pynutil.add_weight(
+                               single_digits_graph_with_commas, 0.0001)
+                           | cardinal_with_leading_zeros).optimize()
+
+            one_to_a_replacement_graph = (
+                pynini.cross("one hundred", "a hundred")
+                | pynini.cross("one thousand", "thousand")
+                | pynini.cross("one million", "a million"))
+            final_graph |= pynini.compose(
+                final_graph,
+                one_to_a_replacement_graph.optimize() +
+                pynini.closure(self.VCHAR)).optimize()
+            # remove commas for 4 digits numbers
+            four_digit_comma_graph = (
+                self.DIGIT - "0") + pynutil.delete(",") + self.DIGIT**3
+            final_graph |= pynini.compose(four_digit_comma_graph.optimize(),
+                                          final_graph).optimize()
+
+        self.final_graph = final_graph
+        optional_minus_graph = pynini.closure(
+            pynutil.insert("negative: ") + pynini.cross("-", "\"true\" "), 0,
+            1)
+        final_graph = optional_minus_graph + pynutil.insert(
+            "integer: \"") + final_graph + pynutil.insert("\"")
+        final_graph = self.add_tokens(final_graph)
+        self.tagger = final_graph.optimize()
+
+    def add_optional_and(self, graph):
+        graph_with_and = graph
+
+        graph_with_and = pynutil.add_weight(graph, 0.00001)
+        not_quote = pynini.closure(self.NOT_QUOTE)
+        no_thousand_million = pynini.difference(
+            not_quote, not_quote + pynini.union("thousand", "million") +
+            not_quote).optimize()
+        integer = (not_quote + pynutil.add_weight(
+            pynini.cross("hundred ", "hundred and ") + no_thousand_million,
+            -0.0001)).optimize()
+
+        no_hundred = pynini.difference(
+            pynini.closure(self.VCHAR),
+            not_quote + pynini.accep("hundred") + not_quote).optimize()
+        integer |= (not_quote + pynutil.add_weight(
+            pynini.cross("thousand ", "thousand and ") + no_hundred,
+            -0.0001)).optimize()
+
+        optional_hundred = pynini.compose((self.DIGIT - "0")**3,
+                                          graph).optimize()
+        optional_hundred = pynini.compose(
+            optional_hundred,
+            pynini.closure(self.VCHAR) + pynini.cross(" hundred", "") +
+            pynini.closure(self.VCHAR))
+        graph_with_and |= pynini.compose(graph, integer).optimize()
+        graph_with_and |= optional_hundred
+        return graph_with_and
+
+    def build_verbalizer(self):
+        """
+        Finite state transducer for verbalizing cardinal, e.g.
+            cardinal { negative: "true" integer: "23" } -> minus twenty three
+        """
+        optional_sign = pynini.cross("negative: \"true\"", "minus ")
+        if not self.deterministic:
+            optional_sign |= pynini.cross("negative: \"true\"", "negative ")
+            optional_sign |= pynini.cross("negative: \"true\"", "dash ")
+
+        self.optional_sign = pynini.closure(optional_sign + self.DELETE_SPACE,
+                                            0, 1)
+
+        integer = pynini.closure(self.NOT_QUOTE)
+
+        self.integer = self.DELETE_SPACE + pynutil.delete(
+            "\"") + integer + pynutil.delete("\"")
+        integer = pynutil.delete("integer:") + self.integer
+
+        self.numbers = self.optional_sign + integer
+        delete_tokens = self.delete_tokens(self.numbers)
+        self.verbalizer = delete_tokens.optimize()