WeTextProcessing 1.0.6.dev19__py3-none-any.whl → 1.2.0__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (82) hide show
  1. itn/chinese/inverse_normalizer.py +30 -20
  2. itn/english/__init__.py +0 -0
  3. itn/english/data/currency.tsv +168 -0
  4. itn/english/data/date_period.tsv +10 -0
  5. itn/english/data/electronic/domain.tsv +10 -0
  6. itn/english/data/electronic/server_name.tsv +17 -0
  7. itn/english/data/electronic/symbols.tsv +22 -0
  8. itn/english/data/electronic/url_symbols.tsv +5 -0
  9. itn/english/data/magnitudes.tsv +4 -0
  10. itn/english/data/measurements.tsv +149 -0
  11. itn/english/data/months.tsv +12 -0
  12. itn/english/data/months_cased.tsv +12 -0
  13. itn/english/data/numbers/digit.tsv +9 -0
  14. itn/english/data/numbers/hundred.tsv +1 -0
  15. itn/english/data/numbers/teen.tsv +10 -0
  16. itn/english/data/numbers/thousands.tsv +22 -0
  17. itn/english/data/numbers/ties.tsv +9 -0
  18. itn/english/data/numbers/zero.tsv +1 -0
  19. itn/english/data/ordinals/digit.tsv +9 -0
  20. itn/english/data/ordinals/teen.tsv +1 -0
  21. itn/english/data/time/minute_to.tsv +59 -0
  22. itn/english/data/time/time_suffix.tsv +8 -0
  23. itn/english/data/time/time_suffix_cased.tsv +8 -0
  24. itn/english/data/time/time_zone.tsv +7 -0
  25. itn/english/data/time/time_zone_cased.tsv +7 -0
  26. itn/english/data/time/to_hour.tsv +12 -0
  27. itn/english/data/whitelist.tsv +417 -0
  28. itn/english/data/whitelist_tech.tsv +450 -0
  29. itn/english/data/year_suffix.tsv +6 -0
  30. itn/english/inverse_normalizer.py +96 -0
  31. itn/english/rules/__init__.py +0 -0
  32. itn/english/rules/cardinal.py +136 -0
  33. itn/english/rules/char.py +29 -0
  34. itn/english/rules/date.py +220 -0
  35. itn/english/rules/decimal.py +85 -0
  36. itn/english/rules/electronic.py +76 -0
  37. itn/english/rules/measure.py +115 -0
  38. itn/english/rules/money.py +145 -0
  39. itn/english/rules/ordinal.py +52 -0
  40. itn/english/rules/punctuation.py +31 -0
  41. itn/english/rules/telephone.py +123 -0
  42. itn/english/rules/time.py +123 -0
  43. itn/english/rules/whitelist.py +32 -0
  44. itn/english/rules/word.py +31 -0
  45. itn/english/test/__init__.py +0 -0
  46. itn/english/test/data/en_cardinal.txt +28 -0
  47. itn/english/test/data/en_date.txt +44 -0
  48. itn/english/test/data/en_decimal.txt +63 -0
  49. itn/english/test/data/en_electronic.txt +25 -0
  50. itn/english/test/data/en_measure.txt +114 -0
  51. itn/english/test/data/en_money.txt +52 -0
  52. itn/english/test/data/en_ordinal.txt +34 -0
  53. itn/english/test/data/en_telephone.txt +24 -0
  54. itn/english/test/data/en_time.txt +36 -0
  55. itn/english/test/data/en_whitelist.txt +12 -0
  56. itn/english/test/data/en_word.txt +55 -0
  57. itn/english/test/normalizer_test.py +43 -0
  58. itn/english/test/utils.py +32 -0
  59. itn/main.py +2 -12
  60. tn/_version.py +2 -2
  61. tn/chinese/data/cardinal/phone.tsv +28 -0
  62. tn/chinese/data/math/operator_unambiguous.tsv +6 -0
  63. tn/chinese/normalizer.py +2 -2
  64. tn/chinese/rules/cardinal.py +13 -3
  65. tn/chinese/rules/math.py +2 -1
  66. tn/chinese/rules/measure.py +5 -5
  67. tn/en_tn_tagger.fst +0 -0
  68. tn/en_tn_verbalizer.fst +0 -0
  69. tn/english/normalizer.py +4 -0
  70. tn/english/rules/serial.py +106 -0
  71. tn/japanese/normalizer.py +0 -1
  72. tn/main.py +2 -2
  73. tn/processor.py +18 -10
  74. tn/token_parser.py +6 -4
  75. tn/utils.py +10 -0
  76. tn/zh_tn_tagger.fst +0 -0
  77. {wetextprocessing-1.0.6.dev19.dist-info → wetextprocessing-1.2.0.dist-info}/METADATA +1 -1
  78. {wetextprocessing-1.0.6.dev19.dist-info → wetextprocessing-1.2.0.dist-info}/RECORD +82 -22
  79. {wetextprocessing-1.0.6.dev19.dist-info → wetextprocessing-1.2.0.dist-info}/WHEEL +0 -0
  80. {wetextprocessing-1.0.6.dev19.dist-info → wetextprocessing-1.2.0.dist-info}/entry_points.txt +0 -0
  81. {wetextprocessing-1.0.6.dev19.dist-info → wetextprocessing-1.2.0.dist-info}/licenses/LICENSE +0 -0
  82. {wetextprocessing-1.0.6.dev19.dist-info → wetextprocessing-1.2.0.dist-info}/top_level.txt +0 -0
@@ -0,0 +1,450 @@
1
+ 10K ten k
2
+ 1D one d
3
+ 1G one g
4
+ 1K one k
5
+ 2.5G two point five g
6
+ 2D two d
7
+ 2G two g
8
+ 2K two k
9
+ 3D three d
10
+ 3G three g
11
+ 4D four d
12
+ 4G four g
13
+ 4K four k
14
+ 5D five d
15
+ 5G five g
16
+ 6G six g
17
+ 7-eleven seven eleven
18
+ 7G seven g
19
+ 8K eight k
20
+ AAA triple a
21
+ AC a c
22
+ ACL a c l
23
+ ADAS a das
24
+ AEB a e b
25
+ AES a e s
26
+ AGX a g x
27
+ AI a i
28
+ AI ai
29
+ AI ais
30
+ ALU a l u
31
+ AMD a m d
32
+ API a p i
33
+ API api
34
+ ARP a r p
35
+ ASAP a s a p
36
+ ASAP a sap
37
+ ASMC a s m c
38
+ ASR a s r
39
+ ASUS a sus
40
+ ASUS asus
41
+ AT&T a t and t
42
+ ATM a t m
43
+ AV1 a v one
44
+ AV1 av one
45
+ AVAS a v a s
46
+ AVAS a vas
47
+ AVRCP a v r c p
48
+ AWS a w s
49
+ AWS aws
50
+ AlphaFold alpha fold
51
+ AstraZeneca astra zeneca
52
+ AstraZeneca astrazeneca
53
+ Audio2Face audio to face
54
+ Audio2Face audio too face
55
+ BBC b b c
56
+ BCP b c p
57
+ BEV b e v
58
+ BGA b g a
59
+ BGP b g p
60
+ BIOS Bios
61
+ BIOS bios
62
+ BIS b i s
63
+ BSP b s p
64
+ BasePOD base pod
65
+ BasePOD basepod
66
+ Billion-X billion X
67
+ C# c sharp
68
+ C++ c plus plus
69
+ CAT5 cat five
70
+ CAT5e cat five e
71
+ CAT6 cat six
72
+ CAT6A cat six a
73
+ CAT7 cat seven
74
+ CAT8 cat eight
75
+ CBC c b c
76
+ CCS c c s
77
+ CCTV c c t v
78
+ CCTV c c tv
79
+ CDN c d n
80
+ CEO c e o
81
+ CES c e s
82
+ CFO c f o
83
+ CHAdeMO cha demo
84
+ CHAdeMO chai demo
85
+ CLI c l i
86
+ CMO c m o
87
+ CMS c m s
88
+ CNN c n n
89
+ COVID covid
90
+ COVID-19 covid nineteen
91
+ CPU c p u
92
+ CSO c s o
93
+ CSP c s p
94
+ CSS c s s
95
+ CV c v
96
+ CVT c v t
97
+ CalTech cal tech
98
+ CalTech caltech
99
+ CapEx cap ex
100
+ CapEx capex
101
+ ChatGPT chat g p t
102
+ CloudXR cloud x r
103
+ ConnectX connect x
104
+ Core 2 core two
105
+ DALI d a l i
106
+ DB d b
107
+ DC d c
108
+ DCDB d c d b
109
+ DDR d d r
110
+ DDoS d d o s
111
+ DDoS d dos
112
+ DGL d g l
113
+ DGX d g x -1.1
114
+ DGX A100 d g x a hundred -1.1
115
+ DGX A100 d g x a one hundred -1.1
116
+ DGX Station d g x station -1.1
117
+ DHCP d h c p
118
+ DLSS d l s s
119
+ DLSS 2 d l s s two
120
+ DLSS 3 d l s s three
121
+ DNA d n a
122
+ DNN d n n
123
+ DNS d n s
124
+ DOS dos
125
+ DP d p
126
+ DPDT d p d t
127
+ DPU d p u
128
+ DPX d p x
129
+ DeepMind deep mind
130
+ DeepMind deepmind
131
+ DeepStream deep stream
132
+ DeepStream deepstream
133
+ DevOps dev ops
134
+ Direct3D direct three d
135
+ DisplayPort display port
136
+ EBC e b c
137
+ EBS e b s
138
+ EC2 e c two
139
+ ECC e c c
140
+ ECS e c s
141
+ EFS e f s
142
+ EGX e g x
143
+ EKS e k s
144
+ ELB e l b
145
+ ELCB e l c b
146
+ EMR e m r
147
+ EOL e o l
148
+ EV e v
149
+ Earth-2 earth two
150
+ FAT12 fat twelve
151
+ FAT16 fat sixteen
152
+ FAT32 fat thirty two
153
+ FIFO fee foe
154
+ FIFO fifo
155
+ FOMO foe moe
156
+ FOMO fomo
157
+ FPS f p s
158
+ FSA f s a
159
+ FST f s t
160
+ FTP f t p
161
+ FTPS f t p s
162
+ FX f x
163
+ G-SYNC g sync
164
+ GCC g c c
165
+ GCP g c p
166
+ GCS g c s
167
+ GFN G f n
168
+ GPL G P L
169
+ GPLv2 G P L two
170
+ GPLv3 G P L three
171
+ GPU g p u
172
+ GRPC g r p c
173
+ GRU g r u
174
+ GSK g s k
175
+ GTC g t c
176
+ GTX g t x -1.1
177
+ GUI g u i
178
+ GUI gui
179
+ GeForce geforce
180
+ GeForce NOW geforce now
181
+ H.264 H two six four
182
+ H.264 H two sixty four
183
+ H.265 H two six five
184
+ H.265 H two sixty five
185
+ H100 H one hundred
186
+ HDMI h d m i
187
+ HEVC h e v c
188
+ HFS h f s
189
+ HFS+ h f s plus
190
+ HGX h g x
191
+ HMD Hmd
192
+ HPC h p c
193
+ HQ h q
194
+ HSM h s m
195
+ HT h t
196
+ HTML h t m l
197
+ HTTP h t t p
198
+ HTTPS h t t p s
199
+ HuggingFace hugging face
200
+ I/O i o
201
+ ICMP i c m p
202
+ IEC i e c
203
+ IEEE i triple e
204
+ IGMP i g m p
205
+ IGX i g x
206
+ IISc i i s c
207
+ IIT i i t
208
+ INT8 int eight
209
+ IP i p
210
+ IPSec i p sec
211
+ ISC i s c
212
+ ISI i s i
213
+ InfiniBand inifiband
214
+ IoT i o t
215
+ Isaac SIM isaac sim
216
+ JSON j son
217
+ JavaScript java script
218
+ JavaScript javascript
219
+ KVM k v m
220
+ LFP lee fei po
221
+ LFP life po
222
+ LGA l g a -1.1
223
+ LLC l l c
224
+ LLM l l m
225
+ LLMs l l ms
226
+ LLP l l p
227
+ LSTM l s t m
228
+ LT l t
229
+ LTT l t t
230
+ LinkedIn linked in
231
+ LinkedIn linkedin
232
+ MB m b
233
+ MCB m c b
234
+ MCCB m c c b
235
+ MCP m c p
236
+ MDM m d m
237
+ MFA m f a
238
+ MIT m i t
239
+ ML m l
240
+ MLPerf m l perf
241
+ MONAI monai
242
+ MWC m w c
243
+ MXNet m x net
244
+ MacBook mac book
245
+ MacBook macbook
246
+ MacBook Pro mac book pro
247
+ Max-Q max q
248
+ Million-X million x
249
+ MoCap mo cap
250
+ NC n c
251
+ NGC n g c
252
+ NLP n l p
253
+ NLU n l u
254
+ NMC n m c
255
+ NO n o
256
+ NTFS n t f s
257
+ NTP n t p
258
+ NVDec n v dec
259
+ NVDec n v deck
260
+ NVEnc n v enc
261
+ NVEnc n v inc
262
+ NVEnc n v ink
263
+ NVMe n v m e
264
+ Ni-CD knee cad
265
+ NiMH n i m h
266
+ NoSQL no s q l
267
+ NoSQL no sql
268
+ Nvidia A100 Nvidia a hundred
269
+ Nvidia A100 Nvidia a one hundred
270
+ Nvidia A100 nvidia a hundred
271
+ Nvidia A100 nvidia a one hundred
272
+ Nvidia A30 Nvidia a thirty
273
+ Nvidia A30 nvidia a thirty
274
+ Nvidia A40 Nvidia a forty
275
+ Nvidia A40 nvidia a forty
276
+ OCI o c i
277
+ OEM o e m
278
+ OEMs o e ms
279
+ OLTC o l t c
280
+ OLTP o l t p
281
+ ONNX o n n x
282
+ OVX o v x
283
+ OpEx op ex
284
+ OpEx opex
285
+ OpenCL open c l
286
+ OpenGL open g l
287
+ PCI p c i
288
+ PCIe p c i e -1.1
289
+ PDF p d f
290
+ PHEV p h e v
291
+ POP3 pop three
292
+ PSO p s o
293
+ Photoshop photo shop
294
+ PhysX fizz ex
295
+ PhysX fizz x
296
+ PhysX phys ex
297
+ PhysX phys x
298
+ PlayStation play station
299
+ Premiere Pro premiere pro
300
+ ProVis pro vis
301
+ ProVis provis
302
+ PyG pi g
303
+ PyG py g
304
+ PyTorch pi torch
305
+ QODA q o d a
306
+ QOS q o s
307
+ RCBO r c b o
308
+ RCCB r c c b
309
+ RDS r d s
310
+ RFP r f p
311
+ RISC risc
312
+ RL r l
313
+ RNN r n n
314
+ ROI r o i
315
+ RSS r s s
316
+ RT r t
317
+ RTP r t p
318
+ RTX r t x -1.1
319
+ RTX ON r t x on
320
+ RTX Off r t x off
321
+ RacerX racer x
322
+ Ryzen 2 rye zen two
323
+ Ryzen 3 rye zen three
324
+ Ryzen 5 rye zen five
325
+ Ryzen 7 rye zen seven
326
+ Ryzen 9 rye zen nine
327
+ S&P s and p
328
+ S3 s three
329
+ SAE s a e
330
+ SDK s d k
331
+ SEO s e o
332
+ SFU s f u
333
+ SHIELD TV shield tv
334
+ SIGGRAPH sig graph
335
+ SIGGRAPH siggraph
336
+ SMPS s m p s
337
+ SMTP s m t p
338
+ SNMP s n m p
339
+ SNS s n s
340
+ SOL s o l
341
+ SPDT s p d t
342
+ SPN s p n
343
+ SPST s p s t
344
+ SQL s q l
345
+ SQS s q s
346
+ SSD s s d
347
+ SSH s s h
348
+ SSL s s l
349
+ SSO s s o
350
+ SSR s s r
351
+ SUV s u v
352
+ SWG s w g
353
+ SaaS saas
354
+ ServiceNow service now
355
+ SuperCloud super cloud
356
+ SuperPOD super pod
357
+ TCP t c p
358
+ TCP/IP t c p i p
359
+ TFTP t f t p
360
+ TJ t j
361
+ TOS t o s
362
+ TPM t p m
363
+ TPMS t p m s
364
+ TPN t p n
365
+ TSL t s l
366
+ TSMC t s m c
367
+ TTS t t s
368
+ Telnet tell net
369
+ TensorRT tensor r t
370
+ The A100 The a hundred
371
+ The A100 The a one hundred
372
+ The A100 the a hundred
373
+ The A100 the a one hundred
374
+ The A30 The a thirty
375
+ The A30 the a thirty
376
+ The A40 The a forty
377
+ The A40 the a forty
378
+ Tick-Tock tick tock
379
+ TikTok app tick tock app
380
+ TikTok is tick tock is
381
+ UDP u d p
382
+ UEFI u e f i
383
+ UFI u f i
384
+ UFS u f s
385
+ UPS u p s
386
+ USB u s b
387
+ VAAPI v a a p i
388
+ VAAPI v a api
389
+ VDI v d i
390
+ VDPAU v d p a u
391
+ VDPAU v d paw
392
+ VDPAU v d pow
393
+ VFD v f d
394
+ VLAN v lan
395
+ VMFS v m f s
396
+ VP v p
397
+ VPN v p n
398
+ Vcc v c c
399
+ Vdd v d d
400
+ VoIP v o i p
401
+ WFSA w f s a
402
+ WFST w f s t
403
+ Wi-Fi wi fi
404
+ Wi-Fi wifi
405
+ XFX x f x
406
+ XFX x fx
407
+ XML x m l
408
+ ZFS z f s
409
+ Zen 2 zen two
410
+ Zen 3 zen three
411
+ Zen 4 zen four
412
+ b-tree b t ree
413
+ btrfs b t r f s
414
+ btrfs b tree f s
415
+ btrfs butter f s
416
+ c# c sharp
417
+ c++ c plus plus
418
+ cuBLAS cue blah
419
+ cuBLAS cue blahs
420
+ cuBLAS q blahs
421
+ cuDNN c u d n n
422
+ cuDNN cue d n n
423
+ cuDNN q d n n
424
+ cuOpt cue opt
425
+ cuOpt q opt
426
+ cuQuantum cue quantum
427
+ cuQuantum q quantum
428
+ dr. doctor
429
+ e.g. for example
430
+ es3 e s three
431
+ exFAT ex fat
432
+ ext3 e x t three
433
+ ext4 e x t four
434
+ i3 i three
435
+ i5 i five
436
+ i7 i seven
437
+ i9 i nine
438
+ iOS ios
439
+ iTPMS i t p m s
440
+ mr. mister
441
+ mrs. misses
442
+ on TikTok on tick tock
443
+ sFTP s f t p
444
+ st. saint
445
+ x1 x one
446
+ x16 x sixteen
447
+ x2 x two
448
+ x4 x four
449
+ x8 x eight
450
+ xfs x f s
@@ -0,0 +1,6 @@
1
+ AD a d
2
+ BC b c
3
+ BCE b c e
4
+ BCE before common era
5
+ CE c e
6
+ CE common era
@@ -0,0 +1,96 @@
1
+ # Copyright (c) 2026 Zhendong Peng (pzd17@tsinghua.org.cn)
2
+ #
3
+ # Licensed under the Apache License, Version 2.0 (the "License");
4
+ # you may not use this file except in compliance with the License.
5
+ # You may obtain a copy of the License at
6
+ #
7
+ # http://www.apache.org/licenses/LICENSE-2.0
8
+ #
9
+ # Unless required by applicable law or agreed to in writing, software
10
+ # distributed under the License is distributed on an "AS IS" BASIS,
11
+ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12
+ # See the License for the specific language governing permissions and
13
+ # limitations under the License.
14
+
15
+ from importlib_resources import files
16
+ from pynini import closure
17
+ from pynini.lib.pynutil import add_weight, delete
18
+
19
+ from itn.english.rules.cardinal import Cardinal
20
+ from itn.english.rules.char import Char
21
+ from itn.english.rules.date import Date
22
+ from itn.english.rules.decimal import Decimal
23
+ from itn.english.rules.electronic import Electronic
24
+ from itn.english.rules.measure import Measure
25
+ from itn.english.rules.money import Money
26
+ from itn.english.rules.ordinal import Ordinal
27
+ from itn.english.rules.punctuation import Punctuation
28
+ from itn.english.rules.telephone import Telephone
29
+ from itn.english.rules.time import Time
30
+ from itn.english.rules.whitelist import Whitelist
31
+ from itn.english.rules.word import Word
32
+ from tn.processor import Processor
33
+
34
+
35
+ class InverseNormalizer(Processor):
36
+
37
+ def __init__(self, cache_dir=None, overwrite_cache=False):
38
+ super().__init__(name="en_inverse_normalizer", ordertype="itn")
39
+ if cache_dir is None:
40
+ cache_dir = files("itn")
41
+ self.build_fst("en_itn", cache_dir, overwrite_cache)
42
+
43
+ def build_tagger_and_verbalizer(self):
44
+ cardinal = Cardinal()
45
+ ordinal = Ordinal(cardinal=cardinal)
46
+ decimal = Decimal(cardinal=cardinal)
47
+ date = Date(cardinal=cardinal, ordinal=ordinal)
48
+ time = Time(cardinal=cardinal)
49
+ measure = Measure(cardinal=cardinal, decimal=decimal)
50
+ money = Money(cardinal=cardinal, decimal=decimal)
51
+ telephone = Telephone(cardinal=cardinal)
52
+ electronic = Electronic()
53
+ whitelist = Whitelist()
54
+ word = Word()
55
+ char = Char()
56
+ punctuation = Punctuation()
57
+
58
+ classify = (
59
+ add_weight(date.tagger, 1.09)
60
+ | add_weight(time.tagger, 1.1)
61
+ | add_weight(measure.tagger, 1.1)
62
+ | add_weight(money.tagger, 1.08)
63
+ | add_weight(whitelist.tagger, 1.01)
64
+ | add_weight(telephone.tagger, 1.1)
65
+ | add_weight(electronic.tagger, 1.1)
66
+ | add_weight(ordinal.tagger, 1.09)
67
+ | add_weight(decimal.tagger, 1.1)
68
+ | add_weight(cardinal.tagger, 1.1)
69
+ | add_weight(word.tagger, 50)
70
+ | add_weight(char.tagger, 100)
71
+ ).optimize()
72
+
73
+ punct = add_weight(punctuation.tagger, 1.1)
74
+ token = closure(punct + delete(" ").ques) + classify + closure(delete(" ").ques + punct)
75
+ graph = token + closure(self.DELETE_EXTRA_SPACE + token)
76
+ self.tagger = delete(" ").star + graph + delete(" ").star
77
+
78
+ verbalizer = (
79
+ cardinal.verbalizer
80
+ | ordinal.verbalizer
81
+ | decimal.verbalizer
82
+ | date.verbalizer
83
+ | time.verbalizer
84
+ | measure.verbalizer
85
+ | money.verbalizer
86
+ | telephone.verbalizer
87
+ | electronic.verbalizer
88
+ | whitelist.verbalizer
89
+ | word.verbalizer
90
+ | char.verbalizer
91
+ | punctuation.verbalizer
92
+ ).optimize()
93
+
94
+ self.verbalizer = (verbalizer + self.INSERT_SPACE).star @ self.build_rule(
95
+ self.DELETE_EXTRA_SPACE
96
+ ) @ self.build_rule(delete(" "), r="[EOS]")
File without changes
@@ -0,0 +1,136 @@
1
+ # Copyright (c) 2026 Zhendong Peng (pzd17@tsinghua.org.cn)
2
+ #
3
+ # Licensed under the Apache License, Version 2.0 (the "License");
4
+ # you may not use this file except in compliance with the License.
5
+ # You may obtain a copy of the License at
6
+ #
7
+ # http://www.apache.org/licenses/LICENSE-2.0
8
+ #
9
+ # Unless required by applicable law or agreed to in writing, software
10
+ # distributed under the License is distributed on an "AS IS" BASIS,
11
+ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12
+ # See the License for the specific language governing permissions and
13
+ # limitations under the License.
14
+
15
+ from pynini import closure, cross, difference, string_file, union
16
+ from pynini.lib.pynutil import add_weight, delete, insert
17
+
18
+ from tn.processor import Processor
19
+ from tn.utils import get_abs_path
20
+
21
+
22
+ class Cardinal(Processor):
23
+
24
+ def __init__(self):
25
+ super().__init__(name="cardinal", ordertype="itn")
26
+ self.build_tagger()
27
+ self.build_verbalizer()
28
+
29
+ def build_tagger(self):
30
+ zero = string_file(get_abs_path("../itn/english/data/numbers/zero.tsv"))
31
+ digit = string_file(get_abs_path("../itn/english/data/numbers/digit.tsv"))
32
+ teen = string_file(get_abs_path("../itn/english/data/numbers/teen.tsv"))
33
+ ties = string_file(get_abs_path("../itn/english/data/numbers/ties.tsv"))
34
+ ds = delete(" ")
35
+
36
+ # 1~9, 10~19, 20~99
37
+ one_digit = digit
38
+ two_digit = teen | (ties + (ds + digit | add_weight(insert("0"), 0.1)))
39
+ self.graph_two_digit = two_digit
40
+ up_to_99 = one_digit | two_digit
41
+
42
+ # one hundred, one hundred twenty three, one hundred one
43
+ hundreds = digit + ds + delete("hundred") + (ds + two_digit | ds + insert("0") + one_digit | insert("00"))
44
+ # eleven hundred => 1100, twenty one hundred => 2100
45
+ hundreds_as_thousand = (teen | ties + ds + digit) + ds + delete("hundred") + (
46
+ ds + two_digit | ds + insert("0") + one_digit | insert("00")
47
+ )
48
+
49
+ # 1~999
50
+ up_to_999 = up_to_99 | hundreds
51
+ self.up_to_999 = up_to_999
52
+ # 1~999 with zero-padding to 3 digits
53
+ up_to_999_padded = hundreds | insert("0") + two_digit | insert("00") + one_digit
54
+
55
+ def _with_mag(name):
56
+ return up_to_999 + ds + delete(name)
57
+
58
+ def _with_mag_padded(name):
59
+ return up_to_999_padded + ds + delete(name)
60
+
61
+ graph = zero | up_to_999 | hundreds_as_thousand
62
+ graph |= _with_mag("thousand") + (ds + up_to_999_padded | insert("000"))
63
+ graph |= (
64
+ _with_mag("million")
65
+ + (ds + _with_mag_padded("thousand") | insert("000"))
66
+ + (ds + up_to_999_padded | insert("000"))
67
+ )
68
+ graph |= (
69
+ _with_mag("billion")
70
+ + (ds + _with_mag_padded("million") | insert("000"))
71
+ + (ds + _with_mag_padded("thousand") | insert("000"))
72
+ + (ds + up_to_999_padded | insert("000"))
73
+ )
74
+ graph |= (
75
+ _with_mag("trillion")
76
+ + (ds + _with_mag_padded("billion") | insert("000"))
77
+ + (ds + _with_mag_padded("million") | insert("000"))
78
+ + (ds + _with_mag_padded("thousand") | insert("000"))
79
+ + (ds + up_to_999_padded | insert("000"))
80
+ )
81
+ graph |= (
82
+ _with_mag("quadrillion")
83
+ + (ds + _with_mag_padded("trillion") | insert("000"))
84
+ + (ds + _with_mag_padded("billion") | insert("000"))
85
+ + (ds + _with_mag_padded("million") | insert("000"))
86
+ + (ds + _with_mag_padded("thousand") | insert("000"))
87
+ + (ds + up_to_999_padded | insert("000"))
88
+ )
89
+ graph |= (
90
+ _with_mag("quintillion")
91
+ + (ds + _with_mag_padded("quadrillion") | insert("000"))
92
+ + (ds + _with_mag_padded("trillion") | insert("000"))
93
+ + (ds + _with_mag_padded("billion") | insert("000"))
94
+ + (ds + _with_mag_padded("million") | insert("000"))
95
+ + (ds + _with_mag_padded("thousand") | insert("000"))
96
+ + (ds + up_to_999_padded | insert("000"))
97
+ )
98
+ graph |= (
99
+ _with_mag("sextillion")
100
+ + (ds + _with_mag_padded("quintillion") | insert("000"))
101
+ + (ds + _with_mag_padded("quadrillion") | insert("000"))
102
+ + (ds + _with_mag_padded("trillion") | insert("000"))
103
+ + (ds + _with_mag_padded("billion") | insert("000"))
104
+ + (ds + _with_mag_padded("million") | insert("000"))
105
+ + (ds + _with_mag_padded("thousand") | insert("000"))
106
+ + (ds + up_to_999_padded | insert("000"))
107
+ )
108
+
109
+ # strip leading zeros
110
+ graph = graph @ union(delete(closure("0")) + (self.DIGIT - "0") + closure(self.DIGIT), "0")
111
+ # delete optional "and"
112
+ delete_and = self.build_rule(delete("and "), " ", self.ALPHA)
113
+ graph = (delete_and @ graph).optimize()
114
+
115
+ self.graph = graph
116
+ self.graph_no_exception = graph
117
+
118
+ # exclude 0-12 from cardinal tagger (they stay as words)
119
+ from itn.english.rules.time import _num_to_word
120
+ exception_labels = ["zero"] + [_num_to_word(x) for x in range(1, 13)]
121
+ exception = union(*exception_labels).optimize()
122
+ graph_with_exception = (difference(self.VSIGMA, exception) @ graph).optimize()
123
+
124
+ minus = delete("minus") | delete("negative")
125
+ optional_minus = closure(insert('negative: "-" ') + minus + ds, 0, 1)
126
+ final_graph = optional_minus + insert('integer: "') + graph_with_exception + insert('"')
127
+ self.tagger = self.add_tokens(final_graph)
128
+
129
+ def build_verbalizer(self):
130
+ optional_sign = closure(
131
+ delete('negative:') + self.DELETE_SPACE + delete('"') + self.NOT_QUOTE + delete('"') + self.DELETE_SPACE,
132
+ 0, 1,
133
+ )
134
+ integer = delete("integer:") + self.DELETE_SPACE + delete('"') + self.NOT_QUOTE.plus + delete('"')
135
+ self.numbers = integer
136
+ self.verbalizer = self.delete_tokens(optional_sign + integer)
@@ -0,0 +1,29 @@
1
+ # Copyright (c) 2026 Zhendong Peng (pzd17@tsinghua.org.cn)
2
+ #
3
+ # Licensed under the Apache License, Version 2.0 (the "License");
4
+ # you may not use this file except in compliance with the License.
5
+ # You may obtain a copy of the License at
6
+ #
7
+ # http://www.apache.org/licenses/LICENSE-2.0
8
+ #
9
+ # Unless required by applicable law or agreed to in writing, software
10
+ # distributed under the License is distributed on an "AS IS" BASIS,
11
+ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12
+ # See the License for the specific language governing permissions and
13
+ # limitations under the License.
14
+
15
+ from pynini.lib.pynutil import insert
16
+
17
+ from tn.processor import Processor
18
+
19
+
20
+ class Char(Processor):
21
+
22
+ def __init__(self):
23
+ super().__init__(name="char", ordertype="itn")
24
+ self.build_tagger()
25
+ self.build_verbalizer()
26
+
27
+ def build_tagger(self):
28
+ tagger = insert('value: "') + self.CHAR + insert('"')
29
+ self.tagger = self.add_tokens(tagger)