WeTextProcessing 1.0.6.dev19__py3-none-any.whl → 1.2.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- itn/chinese/inverse_normalizer.py +30 -20
- itn/english/__init__.py +0 -0
- itn/english/data/currency.tsv +168 -0
- itn/english/data/date_period.tsv +10 -0
- itn/english/data/electronic/domain.tsv +10 -0
- itn/english/data/electronic/server_name.tsv +17 -0
- itn/english/data/electronic/symbols.tsv +22 -0
- itn/english/data/electronic/url_symbols.tsv +5 -0
- itn/english/data/magnitudes.tsv +4 -0
- itn/english/data/measurements.tsv +149 -0
- itn/english/data/months.tsv +12 -0
- itn/english/data/months_cased.tsv +12 -0
- itn/english/data/numbers/digit.tsv +9 -0
- itn/english/data/numbers/hundred.tsv +1 -0
- itn/english/data/numbers/teen.tsv +10 -0
- itn/english/data/numbers/thousands.tsv +22 -0
- itn/english/data/numbers/ties.tsv +9 -0
- itn/english/data/numbers/zero.tsv +1 -0
- itn/english/data/ordinals/digit.tsv +9 -0
- itn/english/data/ordinals/teen.tsv +1 -0
- itn/english/data/time/minute_to.tsv +59 -0
- itn/english/data/time/time_suffix.tsv +8 -0
- itn/english/data/time/time_suffix_cased.tsv +8 -0
- itn/english/data/time/time_zone.tsv +7 -0
- itn/english/data/time/time_zone_cased.tsv +7 -0
- itn/english/data/time/to_hour.tsv +12 -0
- itn/english/data/whitelist.tsv +417 -0
- itn/english/data/whitelist_tech.tsv +450 -0
- itn/english/data/year_suffix.tsv +6 -0
- itn/english/inverse_normalizer.py +96 -0
- itn/english/rules/__init__.py +0 -0
- itn/english/rules/cardinal.py +136 -0
- itn/english/rules/char.py +29 -0
- itn/english/rules/date.py +220 -0
- itn/english/rules/decimal.py +85 -0
- itn/english/rules/electronic.py +76 -0
- itn/english/rules/measure.py +115 -0
- itn/english/rules/money.py +145 -0
- itn/english/rules/ordinal.py +52 -0
- itn/english/rules/punctuation.py +31 -0
- itn/english/rules/telephone.py +123 -0
- itn/english/rules/time.py +123 -0
- itn/english/rules/whitelist.py +32 -0
- itn/english/rules/word.py +31 -0
- itn/english/test/__init__.py +0 -0
- itn/english/test/data/en_cardinal.txt +28 -0
- itn/english/test/data/en_date.txt +44 -0
- itn/english/test/data/en_decimal.txt +63 -0
- itn/english/test/data/en_electronic.txt +25 -0
- itn/english/test/data/en_measure.txt +114 -0
- itn/english/test/data/en_money.txt +52 -0
- itn/english/test/data/en_ordinal.txt +34 -0
- itn/english/test/data/en_telephone.txt +24 -0
- itn/english/test/data/en_time.txt +36 -0
- itn/english/test/data/en_whitelist.txt +12 -0
- itn/english/test/data/en_word.txt +55 -0
- itn/english/test/normalizer_test.py +43 -0
- itn/english/test/utils.py +32 -0
- itn/main.py +2 -12
- tn/_version.py +2 -2
- tn/chinese/data/cardinal/phone.tsv +28 -0
- tn/chinese/data/math/operator_unambiguous.tsv +6 -0
- tn/chinese/normalizer.py +2 -2
- tn/chinese/rules/cardinal.py +13 -3
- tn/chinese/rules/math.py +2 -1
- tn/chinese/rules/measure.py +5 -5
- tn/en_tn_tagger.fst +0 -0
- tn/en_tn_verbalizer.fst +0 -0
- tn/english/normalizer.py +4 -0
- tn/english/rules/serial.py +106 -0
- tn/japanese/normalizer.py +0 -1
- tn/main.py +2 -2
- tn/processor.py +18 -10
- tn/token_parser.py +6 -4
- tn/utils.py +10 -0
- tn/zh_tn_tagger.fst +0 -0
- {wetextprocessing-1.0.6.dev19.dist-info → wetextprocessing-1.2.0.dist-info}/METADATA +1 -1
- {wetextprocessing-1.0.6.dev19.dist-info → wetextprocessing-1.2.0.dist-info}/RECORD +82 -22
- {wetextprocessing-1.0.6.dev19.dist-info → wetextprocessing-1.2.0.dist-info}/WHEEL +0 -0
- {wetextprocessing-1.0.6.dev19.dist-info → wetextprocessing-1.2.0.dist-info}/entry_points.txt +0 -0
- {wetextprocessing-1.0.6.dev19.dist-info → wetextprocessing-1.2.0.dist-info}/licenses/LICENSE +0 -0
- {wetextprocessing-1.0.6.dev19.dist-info → wetextprocessing-1.2.0.dist-info}/top_level.txt +0 -0
|
@@ -0,0 +1,450 @@
|
|
|
1
|
+
10K ten k
|
|
2
|
+
1D one d
|
|
3
|
+
1G one g
|
|
4
|
+
1K one k
|
|
5
|
+
2.5G two point five g
|
|
6
|
+
2D two d
|
|
7
|
+
2G two g
|
|
8
|
+
2K two k
|
|
9
|
+
3D three d
|
|
10
|
+
3G three g
|
|
11
|
+
4D four d
|
|
12
|
+
4G four g
|
|
13
|
+
4K four k
|
|
14
|
+
5D five d
|
|
15
|
+
5G five g
|
|
16
|
+
6G six g
|
|
17
|
+
7-eleven seven eleven
|
|
18
|
+
7G seven g
|
|
19
|
+
8K eight k
|
|
20
|
+
AAA triple a
|
|
21
|
+
AC a c
|
|
22
|
+
ACL a c l
|
|
23
|
+
ADAS a das
|
|
24
|
+
AEB a e b
|
|
25
|
+
AES a e s
|
|
26
|
+
AGX a g x
|
|
27
|
+
AI a i
|
|
28
|
+
AI ai
|
|
29
|
+
AI ais
|
|
30
|
+
ALU a l u
|
|
31
|
+
AMD a m d
|
|
32
|
+
API a p i
|
|
33
|
+
API api
|
|
34
|
+
ARP a r p
|
|
35
|
+
ASAP a s a p
|
|
36
|
+
ASAP a sap
|
|
37
|
+
ASMC a s m c
|
|
38
|
+
ASR a s r
|
|
39
|
+
ASUS a sus
|
|
40
|
+
ASUS asus
|
|
41
|
+
AT&T a t and t
|
|
42
|
+
ATM a t m
|
|
43
|
+
AV1 a v one
|
|
44
|
+
AV1 av one
|
|
45
|
+
AVAS a v a s
|
|
46
|
+
AVAS a vas
|
|
47
|
+
AVRCP a v r c p
|
|
48
|
+
AWS a w s
|
|
49
|
+
AWS aws
|
|
50
|
+
AlphaFold alpha fold
|
|
51
|
+
AstraZeneca astra zeneca
|
|
52
|
+
AstraZeneca astrazeneca
|
|
53
|
+
Audio2Face audio to face
|
|
54
|
+
Audio2Face audio too face
|
|
55
|
+
BBC b b c
|
|
56
|
+
BCP b c p
|
|
57
|
+
BEV b e v
|
|
58
|
+
BGA b g a
|
|
59
|
+
BGP b g p
|
|
60
|
+
BIOS Bios
|
|
61
|
+
BIOS bios
|
|
62
|
+
BIS b i s
|
|
63
|
+
BSP b s p
|
|
64
|
+
BasePOD base pod
|
|
65
|
+
BasePOD basepod
|
|
66
|
+
Billion-X billion X
|
|
67
|
+
C# c sharp
|
|
68
|
+
C++ c plus plus
|
|
69
|
+
CAT5 cat five
|
|
70
|
+
CAT5e cat five e
|
|
71
|
+
CAT6 cat six
|
|
72
|
+
CAT6A cat six a
|
|
73
|
+
CAT7 cat seven
|
|
74
|
+
CAT8 cat eight
|
|
75
|
+
CBC c b c
|
|
76
|
+
CCS c c s
|
|
77
|
+
CCTV c c t v
|
|
78
|
+
CCTV c c tv
|
|
79
|
+
CDN c d n
|
|
80
|
+
CEO c e o
|
|
81
|
+
CES c e s
|
|
82
|
+
CFO c f o
|
|
83
|
+
CHAdeMO cha demo
|
|
84
|
+
CHAdeMO chai demo
|
|
85
|
+
CLI c l i
|
|
86
|
+
CMO c m o
|
|
87
|
+
CMS c m s
|
|
88
|
+
CNN c n n
|
|
89
|
+
COVID covid
|
|
90
|
+
COVID-19 covid nineteen
|
|
91
|
+
CPU c p u
|
|
92
|
+
CSO c s o
|
|
93
|
+
CSP c s p
|
|
94
|
+
CSS c s s
|
|
95
|
+
CV c v
|
|
96
|
+
CVT c v t
|
|
97
|
+
CalTech cal tech
|
|
98
|
+
CalTech caltech
|
|
99
|
+
CapEx cap ex
|
|
100
|
+
CapEx capex
|
|
101
|
+
ChatGPT chat g p t
|
|
102
|
+
CloudXR cloud x r
|
|
103
|
+
ConnectX connect x
|
|
104
|
+
Core 2 core two
|
|
105
|
+
DALI d a l i
|
|
106
|
+
DB d b
|
|
107
|
+
DC d c
|
|
108
|
+
DCDB d c d b
|
|
109
|
+
DDR d d r
|
|
110
|
+
DDoS d d o s
|
|
111
|
+
DDoS d dos
|
|
112
|
+
DGL d g l
|
|
113
|
+
DGX d g x -1.1
|
|
114
|
+
DGX A100 d g x a hundred -1.1
|
|
115
|
+
DGX A100 d g x a one hundred -1.1
|
|
116
|
+
DGX Station d g x station -1.1
|
|
117
|
+
DHCP d h c p
|
|
118
|
+
DLSS d l s s
|
|
119
|
+
DLSS 2 d l s s two
|
|
120
|
+
DLSS 3 d l s s three
|
|
121
|
+
DNA d n a
|
|
122
|
+
DNN d n n
|
|
123
|
+
DNS d n s
|
|
124
|
+
DOS dos
|
|
125
|
+
DP d p
|
|
126
|
+
DPDT d p d t
|
|
127
|
+
DPU d p u
|
|
128
|
+
DPX d p x
|
|
129
|
+
DeepMind deep mind
|
|
130
|
+
DeepMind deepmind
|
|
131
|
+
DeepStream deep stream
|
|
132
|
+
DeepStream deepstream
|
|
133
|
+
DevOps dev ops
|
|
134
|
+
Direct3D direct three d
|
|
135
|
+
DisplayPort display port
|
|
136
|
+
EBC e b c
|
|
137
|
+
EBS e b s
|
|
138
|
+
EC2 e c two
|
|
139
|
+
ECC e c c
|
|
140
|
+
ECS e c s
|
|
141
|
+
EFS e f s
|
|
142
|
+
EGX e g x
|
|
143
|
+
EKS e k s
|
|
144
|
+
ELB e l b
|
|
145
|
+
ELCB e l c b
|
|
146
|
+
EMR e m r
|
|
147
|
+
EOL e o l
|
|
148
|
+
EV e v
|
|
149
|
+
Earth-2 earth two
|
|
150
|
+
FAT12 fat twelve
|
|
151
|
+
FAT16 fat sixteen
|
|
152
|
+
FAT32 fat thirty two
|
|
153
|
+
FIFO fee foe
|
|
154
|
+
FIFO fifo
|
|
155
|
+
FOMO foe moe
|
|
156
|
+
FOMO fomo
|
|
157
|
+
FPS f p s
|
|
158
|
+
FSA f s a
|
|
159
|
+
FST f s t
|
|
160
|
+
FTP f t p
|
|
161
|
+
FTPS f t p s
|
|
162
|
+
FX f x
|
|
163
|
+
G-SYNC g sync
|
|
164
|
+
GCC g c c
|
|
165
|
+
GCP g c p
|
|
166
|
+
GCS g c s
|
|
167
|
+
GFN G f n
|
|
168
|
+
GPL G P L
|
|
169
|
+
GPLv2 G P L two
|
|
170
|
+
GPLv3 G P L three
|
|
171
|
+
GPU g p u
|
|
172
|
+
GRPC g r p c
|
|
173
|
+
GRU g r u
|
|
174
|
+
GSK g s k
|
|
175
|
+
GTC g t c
|
|
176
|
+
GTX g t x -1.1
|
|
177
|
+
GUI g u i
|
|
178
|
+
GUI gui
|
|
179
|
+
GeForce geforce
|
|
180
|
+
GeForce NOW geforce now
|
|
181
|
+
H.264 H two six four
|
|
182
|
+
H.264 H two sixty four
|
|
183
|
+
H.265 H two six five
|
|
184
|
+
H.265 H two sixty five
|
|
185
|
+
H100 H one hundred
|
|
186
|
+
HDMI h d m i
|
|
187
|
+
HEVC h e v c
|
|
188
|
+
HFS h f s
|
|
189
|
+
HFS+ h f s plus
|
|
190
|
+
HGX h g x
|
|
191
|
+
HMD Hmd
|
|
192
|
+
HPC h p c
|
|
193
|
+
HQ h q
|
|
194
|
+
HSM h s m
|
|
195
|
+
HT h t
|
|
196
|
+
HTML h t m l
|
|
197
|
+
HTTP h t t p
|
|
198
|
+
HTTPS h t t p s
|
|
199
|
+
HuggingFace hugging face
|
|
200
|
+
I/O i o
|
|
201
|
+
ICMP i c m p
|
|
202
|
+
IEC i e c
|
|
203
|
+
IEEE i triple e
|
|
204
|
+
IGMP i g m p
|
|
205
|
+
IGX i g x
|
|
206
|
+
IISc i i s c
|
|
207
|
+
IIT i i t
|
|
208
|
+
INT8 int eight
|
|
209
|
+
IP i p
|
|
210
|
+
IPSec i p sec
|
|
211
|
+
ISC i s c
|
|
212
|
+
ISI i s i
|
|
213
|
+
InfiniBand inifiband
|
|
214
|
+
IoT i o t
|
|
215
|
+
Isaac SIM isaac sim
|
|
216
|
+
JSON j son
|
|
217
|
+
JavaScript java script
|
|
218
|
+
JavaScript javascript
|
|
219
|
+
KVM k v m
|
|
220
|
+
LFP lee fei po
|
|
221
|
+
LFP life po
|
|
222
|
+
LGA l g a -1.1
|
|
223
|
+
LLC l l c
|
|
224
|
+
LLM l l m
|
|
225
|
+
LLMs l l ms
|
|
226
|
+
LLP l l p
|
|
227
|
+
LSTM l s t m
|
|
228
|
+
LT l t
|
|
229
|
+
LTT l t t
|
|
230
|
+
LinkedIn linked in
|
|
231
|
+
LinkedIn linkedin
|
|
232
|
+
MB m b
|
|
233
|
+
MCB m c b
|
|
234
|
+
MCCB m c c b
|
|
235
|
+
MCP m c p
|
|
236
|
+
MDM m d m
|
|
237
|
+
MFA m f a
|
|
238
|
+
MIT m i t
|
|
239
|
+
ML m l
|
|
240
|
+
MLPerf m l perf
|
|
241
|
+
MONAI monai
|
|
242
|
+
MWC m w c
|
|
243
|
+
MXNet m x net
|
|
244
|
+
MacBook mac book
|
|
245
|
+
MacBook macbook
|
|
246
|
+
MacBook Pro mac book pro
|
|
247
|
+
Max-Q max q
|
|
248
|
+
Million-X million x
|
|
249
|
+
MoCap mo cap
|
|
250
|
+
NC n c
|
|
251
|
+
NGC n g c
|
|
252
|
+
NLP n l p
|
|
253
|
+
NLU n l u
|
|
254
|
+
NMC n m c
|
|
255
|
+
NO n o
|
|
256
|
+
NTFS n t f s
|
|
257
|
+
NTP n t p
|
|
258
|
+
NVDec n v dec
|
|
259
|
+
NVDec n v deck
|
|
260
|
+
NVEnc n v enc
|
|
261
|
+
NVEnc n v inc
|
|
262
|
+
NVEnc n v ink
|
|
263
|
+
NVMe n v m e
|
|
264
|
+
Ni-CD knee cad
|
|
265
|
+
NiMH n i m h
|
|
266
|
+
NoSQL no s q l
|
|
267
|
+
NoSQL no sql
|
|
268
|
+
Nvidia A100 Nvidia a hundred
|
|
269
|
+
Nvidia A100 Nvidia a one hundred
|
|
270
|
+
Nvidia A100 nvidia a hundred
|
|
271
|
+
Nvidia A100 nvidia a one hundred
|
|
272
|
+
Nvidia A30 Nvidia a thirty
|
|
273
|
+
Nvidia A30 nvidia a thirty
|
|
274
|
+
Nvidia A40 Nvidia a forty
|
|
275
|
+
Nvidia A40 nvidia a forty
|
|
276
|
+
OCI o c i
|
|
277
|
+
OEM o e m
|
|
278
|
+
OEMs o e ms
|
|
279
|
+
OLTC o l t c
|
|
280
|
+
OLTP o l t p
|
|
281
|
+
ONNX o n n x
|
|
282
|
+
OVX o v x
|
|
283
|
+
OpEx op ex
|
|
284
|
+
OpEx opex
|
|
285
|
+
OpenCL open c l
|
|
286
|
+
OpenGL open g l
|
|
287
|
+
PCI p c i
|
|
288
|
+
PCIe p c i e -1.1
|
|
289
|
+
PDF p d f
|
|
290
|
+
PHEV p h e v
|
|
291
|
+
POP3 pop three
|
|
292
|
+
PSO p s o
|
|
293
|
+
Photoshop photo shop
|
|
294
|
+
PhysX fizz ex
|
|
295
|
+
PhysX fizz x
|
|
296
|
+
PhysX phys ex
|
|
297
|
+
PhysX phys x
|
|
298
|
+
PlayStation play station
|
|
299
|
+
Premiere Pro premiere pro
|
|
300
|
+
ProVis pro vis
|
|
301
|
+
ProVis provis
|
|
302
|
+
PyG pi g
|
|
303
|
+
PyG py g
|
|
304
|
+
PyTorch pi torch
|
|
305
|
+
QODA q o d a
|
|
306
|
+
QOS q o s
|
|
307
|
+
RCBO r c b o
|
|
308
|
+
RCCB r c c b
|
|
309
|
+
RDS r d s
|
|
310
|
+
RFP r f p
|
|
311
|
+
RISC risc
|
|
312
|
+
RL r l
|
|
313
|
+
RNN r n n
|
|
314
|
+
ROI r o i
|
|
315
|
+
RSS r s s
|
|
316
|
+
RT r t
|
|
317
|
+
RTP r t p
|
|
318
|
+
RTX r t x -1.1
|
|
319
|
+
RTX ON r t x on
|
|
320
|
+
RTX Off r t x off
|
|
321
|
+
RacerX racer x
|
|
322
|
+
Ryzen 2 rye zen two
|
|
323
|
+
Ryzen 3 rye zen three
|
|
324
|
+
Ryzen 5 rye zen five
|
|
325
|
+
Ryzen 7 rye zen seven
|
|
326
|
+
Ryzen 9 rye zen nine
|
|
327
|
+
S&P s and p
|
|
328
|
+
S3 s three
|
|
329
|
+
SAE s a e
|
|
330
|
+
SDK s d k
|
|
331
|
+
SEO s e o
|
|
332
|
+
SFU s f u
|
|
333
|
+
SHIELD TV shield tv
|
|
334
|
+
SIGGRAPH sig graph
|
|
335
|
+
SIGGRAPH siggraph
|
|
336
|
+
SMPS s m p s
|
|
337
|
+
SMTP s m t p
|
|
338
|
+
SNMP s n m p
|
|
339
|
+
SNS s n s
|
|
340
|
+
SOL s o l
|
|
341
|
+
SPDT s p d t
|
|
342
|
+
SPN s p n
|
|
343
|
+
SPST s p s t
|
|
344
|
+
SQL s q l
|
|
345
|
+
SQS s q s
|
|
346
|
+
SSD s s d
|
|
347
|
+
SSH s s h
|
|
348
|
+
SSL s s l
|
|
349
|
+
SSO s s o
|
|
350
|
+
SSR s s r
|
|
351
|
+
SUV s u v
|
|
352
|
+
SWG s w g
|
|
353
|
+
SaaS saas
|
|
354
|
+
ServiceNow service now
|
|
355
|
+
SuperCloud super cloud
|
|
356
|
+
SuperPOD super pod
|
|
357
|
+
TCP t c p
|
|
358
|
+
TCP/IP t c p i p
|
|
359
|
+
TFTP t f t p
|
|
360
|
+
TJ t j
|
|
361
|
+
TOS t o s
|
|
362
|
+
TPM t p m
|
|
363
|
+
TPMS t p m s
|
|
364
|
+
TPN t p n
|
|
365
|
+
TSL t s l
|
|
366
|
+
TSMC t s m c
|
|
367
|
+
TTS t t s
|
|
368
|
+
Telnet tell net
|
|
369
|
+
TensorRT tensor r t
|
|
370
|
+
The A100 The a hundred
|
|
371
|
+
The A100 The a one hundred
|
|
372
|
+
The A100 the a hundred
|
|
373
|
+
The A100 the a one hundred
|
|
374
|
+
The A30 The a thirty
|
|
375
|
+
The A30 the a thirty
|
|
376
|
+
The A40 The a forty
|
|
377
|
+
The A40 the a forty
|
|
378
|
+
Tick-Tock tick tock
|
|
379
|
+
TikTok app tick tock app
|
|
380
|
+
TikTok is tick tock is
|
|
381
|
+
UDP u d p
|
|
382
|
+
UEFI u e f i
|
|
383
|
+
UFI u f i
|
|
384
|
+
UFS u f s
|
|
385
|
+
UPS u p s
|
|
386
|
+
USB u s b
|
|
387
|
+
VAAPI v a a p i
|
|
388
|
+
VAAPI v a api
|
|
389
|
+
VDI v d i
|
|
390
|
+
VDPAU v d p a u
|
|
391
|
+
VDPAU v d paw
|
|
392
|
+
VDPAU v d pow
|
|
393
|
+
VFD v f d
|
|
394
|
+
VLAN v lan
|
|
395
|
+
VMFS v m f s
|
|
396
|
+
VP v p
|
|
397
|
+
VPN v p n
|
|
398
|
+
Vcc v c c
|
|
399
|
+
Vdd v d d
|
|
400
|
+
VoIP v o i p
|
|
401
|
+
WFSA w f s a
|
|
402
|
+
WFST w f s t
|
|
403
|
+
Wi-Fi wi fi
|
|
404
|
+
Wi-Fi wifi
|
|
405
|
+
XFX x f x
|
|
406
|
+
XFX x fx
|
|
407
|
+
XML x m l
|
|
408
|
+
ZFS z f s
|
|
409
|
+
Zen 2 zen two
|
|
410
|
+
Zen 3 zen three
|
|
411
|
+
Zen 4 zen four
|
|
412
|
+
b-tree b t ree
|
|
413
|
+
btrfs b t r f s
|
|
414
|
+
btrfs b tree f s
|
|
415
|
+
btrfs butter f s
|
|
416
|
+
c# c sharp
|
|
417
|
+
c++ c plus plus
|
|
418
|
+
cuBLAS cue blah
|
|
419
|
+
cuBLAS cue blahs
|
|
420
|
+
cuBLAS q blahs
|
|
421
|
+
cuDNN c u d n n
|
|
422
|
+
cuDNN cue d n n
|
|
423
|
+
cuDNN q d n n
|
|
424
|
+
cuOpt cue opt
|
|
425
|
+
cuOpt q opt
|
|
426
|
+
cuQuantum cue quantum
|
|
427
|
+
cuQuantum q quantum
|
|
428
|
+
dr. doctor
|
|
429
|
+
e.g. for example
|
|
430
|
+
es3 e s three
|
|
431
|
+
exFAT ex fat
|
|
432
|
+
ext3 e x t three
|
|
433
|
+
ext4 e x t four
|
|
434
|
+
i3 i three
|
|
435
|
+
i5 i five
|
|
436
|
+
i7 i seven
|
|
437
|
+
i9 i nine
|
|
438
|
+
iOS ios
|
|
439
|
+
iTPMS i t p m s
|
|
440
|
+
mr. mister
|
|
441
|
+
mrs. misses
|
|
442
|
+
on TikTok on tick tock
|
|
443
|
+
sFTP s f t p
|
|
444
|
+
st. saint
|
|
445
|
+
x1 x one
|
|
446
|
+
x16 x sixteen
|
|
447
|
+
x2 x two
|
|
448
|
+
x4 x four
|
|
449
|
+
x8 x eight
|
|
450
|
+
xfs x f s
|
|
@@ -0,0 +1,96 @@
|
|
|
1
|
+
# Copyright (c) 2026 Zhendong Peng (pzd17@tsinghua.org.cn)
|
|
2
|
+
#
|
|
3
|
+
# Licensed under the Apache License, Version 2.0 (the "License");
|
|
4
|
+
# you may not use this file except in compliance with the License.
|
|
5
|
+
# You may obtain a copy of the License at
|
|
6
|
+
#
|
|
7
|
+
# http://www.apache.org/licenses/LICENSE-2.0
|
|
8
|
+
#
|
|
9
|
+
# Unless required by applicable law or agreed to in writing, software
|
|
10
|
+
# distributed under the License is distributed on an "AS IS" BASIS,
|
|
11
|
+
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
|
12
|
+
# See the License for the specific language governing permissions and
|
|
13
|
+
# limitations under the License.
|
|
14
|
+
|
|
15
|
+
from importlib_resources import files
|
|
16
|
+
from pynini import closure
|
|
17
|
+
from pynini.lib.pynutil import add_weight, delete
|
|
18
|
+
|
|
19
|
+
from itn.english.rules.cardinal import Cardinal
|
|
20
|
+
from itn.english.rules.char import Char
|
|
21
|
+
from itn.english.rules.date import Date
|
|
22
|
+
from itn.english.rules.decimal import Decimal
|
|
23
|
+
from itn.english.rules.electronic import Electronic
|
|
24
|
+
from itn.english.rules.measure import Measure
|
|
25
|
+
from itn.english.rules.money import Money
|
|
26
|
+
from itn.english.rules.ordinal import Ordinal
|
|
27
|
+
from itn.english.rules.punctuation import Punctuation
|
|
28
|
+
from itn.english.rules.telephone import Telephone
|
|
29
|
+
from itn.english.rules.time import Time
|
|
30
|
+
from itn.english.rules.whitelist import Whitelist
|
|
31
|
+
from itn.english.rules.word import Word
|
|
32
|
+
from tn.processor import Processor
|
|
33
|
+
|
|
34
|
+
|
|
35
|
+
class InverseNormalizer(Processor):
|
|
36
|
+
|
|
37
|
+
def __init__(self, cache_dir=None, overwrite_cache=False):
|
|
38
|
+
super().__init__(name="en_inverse_normalizer", ordertype="itn")
|
|
39
|
+
if cache_dir is None:
|
|
40
|
+
cache_dir = files("itn")
|
|
41
|
+
self.build_fst("en_itn", cache_dir, overwrite_cache)
|
|
42
|
+
|
|
43
|
+
def build_tagger_and_verbalizer(self):
|
|
44
|
+
cardinal = Cardinal()
|
|
45
|
+
ordinal = Ordinal(cardinal=cardinal)
|
|
46
|
+
decimal = Decimal(cardinal=cardinal)
|
|
47
|
+
date = Date(cardinal=cardinal, ordinal=ordinal)
|
|
48
|
+
time = Time(cardinal=cardinal)
|
|
49
|
+
measure = Measure(cardinal=cardinal, decimal=decimal)
|
|
50
|
+
money = Money(cardinal=cardinal, decimal=decimal)
|
|
51
|
+
telephone = Telephone(cardinal=cardinal)
|
|
52
|
+
electronic = Electronic()
|
|
53
|
+
whitelist = Whitelist()
|
|
54
|
+
word = Word()
|
|
55
|
+
char = Char()
|
|
56
|
+
punctuation = Punctuation()
|
|
57
|
+
|
|
58
|
+
classify = (
|
|
59
|
+
add_weight(date.tagger, 1.09)
|
|
60
|
+
| add_weight(time.tagger, 1.1)
|
|
61
|
+
| add_weight(measure.tagger, 1.1)
|
|
62
|
+
| add_weight(money.tagger, 1.08)
|
|
63
|
+
| add_weight(whitelist.tagger, 1.01)
|
|
64
|
+
| add_weight(telephone.tagger, 1.1)
|
|
65
|
+
| add_weight(electronic.tagger, 1.1)
|
|
66
|
+
| add_weight(ordinal.tagger, 1.09)
|
|
67
|
+
| add_weight(decimal.tagger, 1.1)
|
|
68
|
+
| add_weight(cardinal.tagger, 1.1)
|
|
69
|
+
| add_weight(word.tagger, 50)
|
|
70
|
+
| add_weight(char.tagger, 100)
|
|
71
|
+
).optimize()
|
|
72
|
+
|
|
73
|
+
punct = add_weight(punctuation.tagger, 1.1)
|
|
74
|
+
token = closure(punct + delete(" ").ques) + classify + closure(delete(" ").ques + punct)
|
|
75
|
+
graph = token + closure(self.DELETE_EXTRA_SPACE + token)
|
|
76
|
+
self.tagger = delete(" ").star + graph + delete(" ").star
|
|
77
|
+
|
|
78
|
+
verbalizer = (
|
|
79
|
+
cardinal.verbalizer
|
|
80
|
+
| ordinal.verbalizer
|
|
81
|
+
| decimal.verbalizer
|
|
82
|
+
| date.verbalizer
|
|
83
|
+
| time.verbalizer
|
|
84
|
+
| measure.verbalizer
|
|
85
|
+
| money.verbalizer
|
|
86
|
+
| telephone.verbalizer
|
|
87
|
+
| electronic.verbalizer
|
|
88
|
+
| whitelist.verbalizer
|
|
89
|
+
| word.verbalizer
|
|
90
|
+
| char.verbalizer
|
|
91
|
+
| punctuation.verbalizer
|
|
92
|
+
).optimize()
|
|
93
|
+
|
|
94
|
+
self.verbalizer = (verbalizer + self.INSERT_SPACE).star @ self.build_rule(
|
|
95
|
+
self.DELETE_EXTRA_SPACE
|
|
96
|
+
) @ self.build_rule(delete(" "), r="[EOS]")
|
|
File without changes
|
|
@@ -0,0 +1,136 @@
|
|
|
1
|
+
# Copyright (c) 2026 Zhendong Peng (pzd17@tsinghua.org.cn)
|
|
2
|
+
#
|
|
3
|
+
# Licensed under the Apache License, Version 2.0 (the "License");
|
|
4
|
+
# you may not use this file except in compliance with the License.
|
|
5
|
+
# You may obtain a copy of the License at
|
|
6
|
+
#
|
|
7
|
+
# http://www.apache.org/licenses/LICENSE-2.0
|
|
8
|
+
#
|
|
9
|
+
# Unless required by applicable law or agreed to in writing, software
|
|
10
|
+
# distributed under the License is distributed on an "AS IS" BASIS,
|
|
11
|
+
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
|
12
|
+
# See the License for the specific language governing permissions and
|
|
13
|
+
# limitations under the License.
|
|
14
|
+
|
|
15
|
+
from pynini import closure, cross, difference, string_file, union
|
|
16
|
+
from pynini.lib.pynutil import add_weight, delete, insert
|
|
17
|
+
|
|
18
|
+
from tn.processor import Processor
|
|
19
|
+
from tn.utils import get_abs_path
|
|
20
|
+
|
|
21
|
+
|
|
22
|
+
class Cardinal(Processor):
|
|
23
|
+
|
|
24
|
+
def __init__(self):
|
|
25
|
+
super().__init__(name="cardinal", ordertype="itn")
|
|
26
|
+
self.build_tagger()
|
|
27
|
+
self.build_verbalizer()
|
|
28
|
+
|
|
29
|
+
def build_tagger(self):
|
|
30
|
+
zero = string_file(get_abs_path("../itn/english/data/numbers/zero.tsv"))
|
|
31
|
+
digit = string_file(get_abs_path("../itn/english/data/numbers/digit.tsv"))
|
|
32
|
+
teen = string_file(get_abs_path("../itn/english/data/numbers/teen.tsv"))
|
|
33
|
+
ties = string_file(get_abs_path("../itn/english/data/numbers/ties.tsv"))
|
|
34
|
+
ds = delete(" ")
|
|
35
|
+
|
|
36
|
+
# 1~9, 10~19, 20~99
|
|
37
|
+
one_digit = digit
|
|
38
|
+
two_digit = teen | (ties + (ds + digit | add_weight(insert("0"), 0.1)))
|
|
39
|
+
self.graph_two_digit = two_digit
|
|
40
|
+
up_to_99 = one_digit | two_digit
|
|
41
|
+
|
|
42
|
+
# one hundred, one hundred twenty three, one hundred one
|
|
43
|
+
hundreds = digit + ds + delete("hundred") + (ds + two_digit | ds + insert("0") + one_digit | insert("00"))
|
|
44
|
+
# eleven hundred => 1100, twenty one hundred => 2100
|
|
45
|
+
hundreds_as_thousand = (teen | ties + ds + digit) + ds + delete("hundred") + (
|
|
46
|
+
ds + two_digit | ds + insert("0") + one_digit | insert("00")
|
|
47
|
+
)
|
|
48
|
+
|
|
49
|
+
# 1~999
|
|
50
|
+
up_to_999 = up_to_99 | hundreds
|
|
51
|
+
self.up_to_999 = up_to_999
|
|
52
|
+
# 1~999 with zero-padding to 3 digits
|
|
53
|
+
up_to_999_padded = hundreds | insert("0") + two_digit | insert("00") + one_digit
|
|
54
|
+
|
|
55
|
+
def _with_mag(name):
|
|
56
|
+
return up_to_999 + ds + delete(name)
|
|
57
|
+
|
|
58
|
+
def _with_mag_padded(name):
|
|
59
|
+
return up_to_999_padded + ds + delete(name)
|
|
60
|
+
|
|
61
|
+
graph = zero | up_to_999 | hundreds_as_thousand
|
|
62
|
+
graph |= _with_mag("thousand") + (ds + up_to_999_padded | insert("000"))
|
|
63
|
+
graph |= (
|
|
64
|
+
_with_mag("million")
|
|
65
|
+
+ (ds + _with_mag_padded("thousand") | insert("000"))
|
|
66
|
+
+ (ds + up_to_999_padded | insert("000"))
|
|
67
|
+
)
|
|
68
|
+
graph |= (
|
|
69
|
+
_with_mag("billion")
|
|
70
|
+
+ (ds + _with_mag_padded("million") | insert("000"))
|
|
71
|
+
+ (ds + _with_mag_padded("thousand") | insert("000"))
|
|
72
|
+
+ (ds + up_to_999_padded | insert("000"))
|
|
73
|
+
)
|
|
74
|
+
graph |= (
|
|
75
|
+
_with_mag("trillion")
|
|
76
|
+
+ (ds + _with_mag_padded("billion") | insert("000"))
|
|
77
|
+
+ (ds + _with_mag_padded("million") | insert("000"))
|
|
78
|
+
+ (ds + _with_mag_padded("thousand") | insert("000"))
|
|
79
|
+
+ (ds + up_to_999_padded | insert("000"))
|
|
80
|
+
)
|
|
81
|
+
graph |= (
|
|
82
|
+
_with_mag("quadrillion")
|
|
83
|
+
+ (ds + _with_mag_padded("trillion") | insert("000"))
|
|
84
|
+
+ (ds + _with_mag_padded("billion") | insert("000"))
|
|
85
|
+
+ (ds + _with_mag_padded("million") | insert("000"))
|
|
86
|
+
+ (ds + _with_mag_padded("thousand") | insert("000"))
|
|
87
|
+
+ (ds + up_to_999_padded | insert("000"))
|
|
88
|
+
)
|
|
89
|
+
graph |= (
|
|
90
|
+
_with_mag("quintillion")
|
|
91
|
+
+ (ds + _with_mag_padded("quadrillion") | insert("000"))
|
|
92
|
+
+ (ds + _with_mag_padded("trillion") | insert("000"))
|
|
93
|
+
+ (ds + _with_mag_padded("billion") | insert("000"))
|
|
94
|
+
+ (ds + _with_mag_padded("million") | insert("000"))
|
|
95
|
+
+ (ds + _with_mag_padded("thousand") | insert("000"))
|
|
96
|
+
+ (ds + up_to_999_padded | insert("000"))
|
|
97
|
+
)
|
|
98
|
+
graph |= (
|
|
99
|
+
_with_mag("sextillion")
|
|
100
|
+
+ (ds + _with_mag_padded("quintillion") | insert("000"))
|
|
101
|
+
+ (ds + _with_mag_padded("quadrillion") | insert("000"))
|
|
102
|
+
+ (ds + _with_mag_padded("trillion") | insert("000"))
|
|
103
|
+
+ (ds + _with_mag_padded("billion") | insert("000"))
|
|
104
|
+
+ (ds + _with_mag_padded("million") | insert("000"))
|
|
105
|
+
+ (ds + _with_mag_padded("thousand") | insert("000"))
|
|
106
|
+
+ (ds + up_to_999_padded | insert("000"))
|
|
107
|
+
)
|
|
108
|
+
|
|
109
|
+
# strip leading zeros
|
|
110
|
+
graph = graph @ union(delete(closure("0")) + (self.DIGIT - "0") + closure(self.DIGIT), "0")
|
|
111
|
+
# delete optional "and"
|
|
112
|
+
delete_and = self.build_rule(delete("and "), " ", self.ALPHA)
|
|
113
|
+
graph = (delete_and @ graph).optimize()
|
|
114
|
+
|
|
115
|
+
self.graph = graph
|
|
116
|
+
self.graph_no_exception = graph
|
|
117
|
+
|
|
118
|
+
# exclude 0-12 from cardinal tagger (they stay as words)
|
|
119
|
+
from itn.english.rules.time import _num_to_word
|
|
120
|
+
exception_labels = ["zero"] + [_num_to_word(x) for x in range(1, 13)]
|
|
121
|
+
exception = union(*exception_labels).optimize()
|
|
122
|
+
graph_with_exception = (difference(self.VSIGMA, exception) @ graph).optimize()
|
|
123
|
+
|
|
124
|
+
minus = delete("minus") | delete("negative")
|
|
125
|
+
optional_minus = closure(insert('negative: "-" ') + minus + ds, 0, 1)
|
|
126
|
+
final_graph = optional_minus + insert('integer: "') + graph_with_exception + insert('"')
|
|
127
|
+
self.tagger = self.add_tokens(final_graph)
|
|
128
|
+
|
|
129
|
+
def build_verbalizer(self):
|
|
130
|
+
optional_sign = closure(
|
|
131
|
+
delete('negative:') + self.DELETE_SPACE + delete('"') + self.NOT_QUOTE + delete('"') + self.DELETE_SPACE,
|
|
132
|
+
0, 1,
|
|
133
|
+
)
|
|
134
|
+
integer = delete("integer:") + self.DELETE_SPACE + delete('"') + self.NOT_QUOTE.plus + delete('"')
|
|
135
|
+
self.numbers = integer
|
|
136
|
+
self.verbalizer = self.delete_tokens(optional_sign + integer)
|
|
@@ -0,0 +1,29 @@
|
|
|
1
|
+
# Copyright (c) 2026 Zhendong Peng (pzd17@tsinghua.org.cn)
|
|
2
|
+
#
|
|
3
|
+
# Licensed under the Apache License, Version 2.0 (the "License");
|
|
4
|
+
# you may not use this file except in compliance with the License.
|
|
5
|
+
# You may obtain a copy of the License at
|
|
6
|
+
#
|
|
7
|
+
# http://www.apache.org/licenses/LICENSE-2.0
|
|
8
|
+
#
|
|
9
|
+
# Unless required by applicable law or agreed to in writing, software
|
|
10
|
+
# distributed under the License is distributed on an "AS IS" BASIS,
|
|
11
|
+
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
|
12
|
+
# See the License for the specific language governing permissions and
|
|
13
|
+
# limitations under the License.
|
|
14
|
+
|
|
15
|
+
from pynini.lib.pynutil import insert
|
|
16
|
+
|
|
17
|
+
from tn.processor import Processor
|
|
18
|
+
|
|
19
|
+
|
|
20
|
+
class Char(Processor):
|
|
21
|
+
|
|
22
|
+
def __init__(self):
|
|
23
|
+
super().__init__(name="char", ordertype="itn")
|
|
24
|
+
self.build_tagger()
|
|
25
|
+
self.build_verbalizer()
|
|
26
|
+
|
|
27
|
+
def build_tagger(self):
|
|
28
|
+
tagger = insert('value: "') + self.CHAR + insert('"')
|
|
29
|
+
self.tagger = self.add_tokens(tagger)
|