WeTextProcessing 1.0.6.dev19__py3-none-any.whl → 1.2.0__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (82) hide show
  1. itn/chinese/inverse_normalizer.py +30 -20
  2. itn/english/__init__.py +0 -0
  3. itn/english/data/currency.tsv +168 -0
  4. itn/english/data/date_period.tsv +10 -0
  5. itn/english/data/electronic/domain.tsv +10 -0
  6. itn/english/data/electronic/server_name.tsv +17 -0
  7. itn/english/data/electronic/symbols.tsv +22 -0
  8. itn/english/data/electronic/url_symbols.tsv +5 -0
  9. itn/english/data/magnitudes.tsv +4 -0
  10. itn/english/data/measurements.tsv +149 -0
  11. itn/english/data/months.tsv +12 -0
  12. itn/english/data/months_cased.tsv +12 -0
  13. itn/english/data/numbers/digit.tsv +9 -0
  14. itn/english/data/numbers/hundred.tsv +1 -0
  15. itn/english/data/numbers/teen.tsv +10 -0
  16. itn/english/data/numbers/thousands.tsv +22 -0
  17. itn/english/data/numbers/ties.tsv +9 -0
  18. itn/english/data/numbers/zero.tsv +1 -0
  19. itn/english/data/ordinals/digit.tsv +9 -0
  20. itn/english/data/ordinals/teen.tsv +1 -0
  21. itn/english/data/time/minute_to.tsv +59 -0
  22. itn/english/data/time/time_suffix.tsv +8 -0
  23. itn/english/data/time/time_suffix_cased.tsv +8 -0
  24. itn/english/data/time/time_zone.tsv +7 -0
  25. itn/english/data/time/time_zone_cased.tsv +7 -0
  26. itn/english/data/time/to_hour.tsv +12 -0
  27. itn/english/data/whitelist.tsv +417 -0
  28. itn/english/data/whitelist_tech.tsv +450 -0
  29. itn/english/data/year_suffix.tsv +6 -0
  30. itn/english/inverse_normalizer.py +96 -0
  31. itn/english/rules/__init__.py +0 -0
  32. itn/english/rules/cardinal.py +136 -0
  33. itn/english/rules/char.py +29 -0
  34. itn/english/rules/date.py +220 -0
  35. itn/english/rules/decimal.py +85 -0
  36. itn/english/rules/electronic.py +76 -0
  37. itn/english/rules/measure.py +115 -0
  38. itn/english/rules/money.py +145 -0
  39. itn/english/rules/ordinal.py +52 -0
  40. itn/english/rules/punctuation.py +31 -0
  41. itn/english/rules/telephone.py +123 -0
  42. itn/english/rules/time.py +123 -0
  43. itn/english/rules/whitelist.py +32 -0
  44. itn/english/rules/word.py +31 -0
  45. itn/english/test/__init__.py +0 -0
  46. itn/english/test/data/en_cardinal.txt +28 -0
  47. itn/english/test/data/en_date.txt +44 -0
  48. itn/english/test/data/en_decimal.txt +63 -0
  49. itn/english/test/data/en_electronic.txt +25 -0
  50. itn/english/test/data/en_measure.txt +114 -0
  51. itn/english/test/data/en_money.txt +52 -0
  52. itn/english/test/data/en_ordinal.txt +34 -0
  53. itn/english/test/data/en_telephone.txt +24 -0
  54. itn/english/test/data/en_time.txt +36 -0
  55. itn/english/test/data/en_whitelist.txt +12 -0
  56. itn/english/test/data/en_word.txt +55 -0
  57. itn/english/test/normalizer_test.py +43 -0
  58. itn/english/test/utils.py +32 -0
  59. itn/main.py +2 -12
  60. tn/_version.py +2 -2
  61. tn/chinese/data/cardinal/phone.tsv +28 -0
  62. tn/chinese/data/math/operator_unambiguous.tsv +6 -0
  63. tn/chinese/normalizer.py +2 -2
  64. tn/chinese/rules/cardinal.py +13 -3
  65. tn/chinese/rules/math.py +2 -1
  66. tn/chinese/rules/measure.py +5 -5
  67. tn/en_tn_tagger.fst +0 -0
  68. tn/en_tn_verbalizer.fst +0 -0
  69. tn/english/normalizer.py +4 -0
  70. tn/english/rules/serial.py +106 -0
  71. tn/japanese/normalizer.py +0 -1
  72. tn/main.py +2 -2
  73. tn/processor.py +18 -10
  74. tn/token_parser.py +6 -4
  75. tn/utils.py +10 -0
  76. tn/zh_tn_tagger.fst +0 -0
  77. {wetextprocessing-1.0.6.dev19.dist-info → wetextprocessing-1.2.0.dist-info}/METADATA +1 -1
  78. {wetextprocessing-1.0.6.dev19.dist-info → wetextprocessing-1.2.0.dist-info}/RECORD +82 -22
  79. {wetextprocessing-1.0.6.dev19.dist-info → wetextprocessing-1.2.0.dist-info}/WHEEL +0 -0
  80. {wetextprocessing-1.0.6.dev19.dist-info → wetextprocessing-1.2.0.dist-info}/entry_points.txt +0 -0
  81. {wetextprocessing-1.0.6.dev19.dist-info → wetextprocessing-1.2.0.dist-info}/licenses/LICENSE +0 -0
  82. {wetextprocessing-1.0.6.dev19.dist-info → wetextprocessing-1.2.0.dist-info}/top_level.txt +0 -0
itn/english/data/whitelist.tsv ADDED
@@ -0,0 +1,417 @@
1
+ 10K ten k
2
+ 1D one d
3
+ 1G one g
4
+ 1K one k
5
+ 2.5G two point five g
6
+ 2D two d
7
+ 2G two g
8
+ 2K two k
9
+ 3D three d
10
+ 3G three g
11
+ 4D four d
12
+ 4G four g
13
+ 4K four k
14
+ 5D five d
15
+ 5G five g
16
+ 6G six g
17
+ 7-eleven seven eleven
18
+ 7G seven g
19
+ 8K eight k
20
+ AAA triple a
21
+ AC a c
22
+ ACL a c l
23
+ ADAS a das
24
+ AEB a e b
25
+ AES a e s
26
+ AGX a g x
27
+ AI a i
28
+ AI ai
29
+ AI ais
30
+ ALU a l u
31
+ AMD a m d
32
+ API a p i
33
+ API api
34
+ ARP a r p
35
+ ASAP a s a p
36
+ ASAP a sap
37
+ ASMC a s m c
38
+ ASR a s r
39
+ ASUS a sus
40
+ ASUS asus
41
+ AT&T a t and t
42
+ ATM a t m
43
+ AV1 a v one
44
+ AV1 av one
45
+ AVAS a v a s
46
+ AVAS a vas
47
+ AVRCP a v r c p
48
+ AWS a w s
49
+ AWS aws
50
+ AlphaFold alpha fold
51
+ AstraZeneca astra zeneca
52
+ AstraZeneca astrazeneca
53
+ Audio2Face audio to face
54
+ Audio2Face audio too face
55
+ BBC b b c
56
+ BCP b c p
57
+ BEV b e v
58
+ BGA b g a
59
+ BGP b g p
60
+ BIOS Bios
61
+ BIOS bios
62
+ BIS b i s
63
+ BSP b s p
64
+ BasePOD base pod
65
+ BasePOD basepod
66
+ Billion-X billion X
67
+ C# c sharp
68
+ C++ c plus plus
69
+ CAT5 cat five
70
+ CAT5e cat five e
71
+ CAT6 cat six
72
+ CAT6A cat six a
73
+ CAT7 cat seven
74
+ CAT8 cat eight
75
+ CBC c b c
76
+ CCS c c s
77
+ CCTV c c t v
78
+ CCTV c c tv
79
+ CDN c d n
80
+ CEO c e o
81
+ CES c e s
82
+ CFO c f o
83
+ CHAdeMO cha demo
84
+ CHAdeMO chai demo
85
+ CLI c l i
86
+ CMO c m o
87
+ CMS c m s
88
+ CNN c n n
89
+ COVID covid
90
+ COVID-19 covid nineteen
91
+ CPU c p u
92
+ CSO c s o
93
+ CSP c s p
94
+ CSS c s s
95
+ CV c v
96
+ CVT c v t
97
+ CalTech cal tech
98
+ CalTech caltech
99
+ CapEx cap ex
100
+ CapEx capex
101
+ ChatGPT chat g p t
102
+ CloudXR cloud x r
103
+ ConnectX connect x
104
+ Core 2 core two
105
+ DALI d a l i
106
+ DB d b
107
+ DC d c
108
+ DCDB d c d b
109
+ DDR d d r
110
+ DDoS d d o s
111
+ DDoS d dos
112
+ DGL d g l
113
+ DGX d g x -1.1
114
+ DGX A100 d g x a hundred -1.1
115
+ DGX A100 d g x a one hundred -1.1
116
+ DGX Station d g x station -1.1
117
+ DHCP d h c p
118
+ DLSS d l s s
119
+ DLSS 2 d l s s two
120
+ DLSS 3 d l s s three
121
+ DNA d n a
122
+ DNN d n n
123
+ DNS d n s
124
+ DOS dos
125
+ DP d p
126
+ DPDT d p d t
127
+ DPU d p u
128
+ DPX d p x
129
+ DeepMind deep mind
130
+ DeepMind deepmind
131
+ DeepStream deep stream
132
+ DeepStream deepstream
133
+ DevOps dev ops
134
+ Direct3D direct three d
135
+ DisplayPort display port
136
+ EBC e b c
137
+ EBS e b s
138
+ EC2 e c two
139
+ ECC e c c
140
+ ECS e c s
141
+ EFS e f s
142
+ EGX e g x
143
+ EKS e k s
144
+ ELB e l b
145
+ ELCB e l c b
146
+ EMR e m r
147
+ EOL e o l
148
+ EV e v
149
+ Earth-2 earth two
150
+ FIFO fee foe
151
+ FIFO fifo
152
+ FOMO foe moe
153
+ FOMO fomo
154
+ FPS f p s
155
+ FSA f s a
156
+ FST f s t
157
+ FTP f t p
158
+ FTPS f t p s
159
+ FX f x
160
+ G-SYNC g sync
161
+ GCC g c c
162
+ GCP g c p
163
+ GCS g c s
164
+ GFN G f n
165
+ GPU g p u
166
+ GRPC g r p c
167
+ GRU g r u
168
+ GSK g s k
169
+ GTC g t c
170
+ GTX g t x -1.1
171
+ GUI g u i
172
+ GUI gui
173
+ GeForce geforce
174
+ GeForce NOW geforce now
175
+ H.264 H two six four
176
+ H.264 H two sixty four
177
+ H.265 H two six five
178
+ H.265 H two sixty five
179
+ H100 H one hundred
180
+ HDMI h d m i
181
+ HEVC h e v c
182
+ HGX h g x
183
+ HMD Hmd
184
+ HPC h p c
185
+ HQ h q
186
+ HSM h s m
187
+ HT h t
188
+ HTML h t m l
189
+ HTTP h t t p
190
+ HTTPS h t t p s
191
+ HuggingFace hugging face
192
+ I/O i o
193
+ ICMP i c m p
194
+ IEC i e c
195
+ IEEE i triple e
196
+ IGMP i g m p
197
+ IGX i g x
198
+ IISc i i s c
199
+ IIT i i t
200
+ INT8 int eight
201
+ IP i p
202
+ IPSec i p sec
203
+ ISC i s c
204
+ ISI i s i
205
+ InfiniBand inifiband
206
+ IoT i o t
207
+ Isaac SIM isaac sim
208
+ JSON j son
209
+ JavaScript java script
210
+ JavaScript javascript
211
+ KVM k v m
212
+ LFP lee fei po
213
+ LFP life po
214
+ LGA l g a -1.1
215
+ LLC l l c
216
+ LLM l l m
217
+ LLMs l l ms
218
+ LLP l l p
219
+ LSTM l s t m
220
+ LT l t
221
+ LTT l t t
222
+ LinkedIn linked in
223
+ LinkedIn linkedin
224
+ MB m b
225
+ MCB m c b
226
+ MCCB m c c b
227
+ MCP m c p
228
+ MDM m d m
229
+ MFA m f a
230
+ MIT m i t
231
+ ML m l
232
+ MLPerf m l perf
233
+ MONAI monai
234
+ MWC m w c
235
+ MXNet m x net
236
+ MacBook mac book
237
+ MacBook macbook
238
+ MacBook Pro mac book pro
239
+ Max-Q max q
240
+ Million-X million x
241
+ MoCap mo cap
242
+ NC n c
243
+ NGC n g c
244
+ NLP n l p
245
+ NLU n l u
246
+ NMC n m c
247
+ NO n o
248
+ NTP n t p
249
+ NVDec n v dec
250
+ NVDec n v deck
251
+ NVEnc n v enc
252
+ NVEnc n v inc
253
+ NVEnc n v ink
254
+ NVMe n v m e
255
+ Ni-CD knee cad
256
+ NiMH n i m h
257
+ NoSQL no s q l
258
+ NoSQL no sql
259
+ Nvidia A100 nvidia a hundred
260
+ Nvidia A100 nvidia a one hundred
261
+ Nvidia A30 nvidia a thirty
262
+ Nvidia A40 nvidia a forty
263
+ OCI o c i
264
+ OEM o e m
265
+ OEMs o e ms
266
+ OLTC o l t c
267
+ OLTP o l t p
268
+ ONNX o n n x
269
+ OVX o v x
270
+ OpEx op ex
271
+ OpEx opex
272
+ OpenCL open c l
273
+ OpenGL open g l
274
+ PCI p c i
275
+ PCIe p c i e -1.1
276
+ PDF p d f
277
+ PHEV p h e v
278
+ POP3 pop three
279
+ PSO p s o
280
+ Photoshop photo shop
281
+ PhysX fizz ex
282
+ PhysX fizz x
283
+ PhysX phys ex
284
+ PhysX phys x
285
+ PlayStation play station
286
+ Premiere Pro premiere pro
287
+ ProVis pro vis
288
+ ProVis provis
289
+ PyG pi g
290
+ PyG py g
291
+ PyTorch pi torch
292
+ QODA q o d a
293
+ QOS q o s
294
+ RCBO r c b o
295
+ RCCB r c c b
296
+ RDS r d s
297
+ RFP r f p
298
+ RISC risc
299
+ RL r l
300
+ RNN r n n
301
+ ROI r o i
302
+ RSS r s s
303
+ RT r t
304
+ RTP r t p
305
+ RTX r t x -1.1
306
+ RTX ON r t x on
307
+ RTX Off r t x off
308
+ RacerX racer x
309
+ Ryzen 2 rye zen two
310
+ Ryzen 3 rye zen three
311
+ Ryzen 5 rye zen five
312
+ Ryzen 7 rye zen seven
313
+ Ryzen 9 rye zen nine
314
+ S&P s and p
315
+ S3 s three
316
+ SAE s a e
317
+ SDK s d k
318
+ SEO s e o
319
+ SFU s f u
320
+ SHIELD TV shield tv
321
+ SIGGRAPH sig graph
322
+ SIGGRAPH siggraph
323
+ SMPS s m p s
324
+ SMTP s m t p
325
+ SNMP s n m p
326
+ SNS s n s
327
+ SOL s o l
328
+ SPDT s p d t
329
+ SPN s p n
330
+ SPST s p s t
331
+ SQL s q l
332
+ SQS s q s
333
+ SSD s s d
334
+ SSH s s h
335
+ SSL s s l
336
+ SSO s s o
337
+ SSR s s r
338
+ SUV s u v
339
+ SWG s w g
340
+ SaaS saas
341
+ ServiceNow service now
342
+ SuperCloud super cloud
343
+ SuperPOD super pod
344
+ TCP t c p
345
+ TCP/IP t c p i p
346
+ TFTP t f t p
347
+ TJ t j
348
+ TOS t o s
349
+ TPM t p m
350
+ TPMS t p m s
351
+ TPN t p n
352
+ TSL t s l
353
+ TSMC t s m c
354
+ TTS t t s
355
+ Telnet tell net
356
+ TensorRT tensor r t
357
+ Tick-Tock tick tock
358
+ TikTok app tick tock app
359
+ TikTok is tick tock is
360
+ UDP u d p
361
+ UEFI u e f i
362
+ UFI u f i
363
+ UPS u p s
364
+ USB u s b
365
+ VAAPI v a a p i
366
+ VAAPI v a api
367
+ VDI v d i
368
+ VDPAU v d p a u
369
+ VDPAU v d paw
370
+ VDPAU v d pow
371
+ VFD v f d
372
+ VLAN v lan
373
+ VP v p
374
+ VPN v p n
375
+ Vcc v c c
376
+ Vdd v d d
377
+ VoIP v o i p
378
+ WFSA w f s a
379
+ WFST w f s t
380
+ Wi-Fi wi fi
381
+ Wi-Fi wifi
382
+ XFX x f x
383
+ XFX x fx
384
+ XML x m l
385
+ Zen 2 zen two
386
+ Zen 3 zen three
387
+ Zen 4 zen four
388
+ c# c sharp
389
+ c++ c plus plus
390
+ cuBLAS cue blah
391
+ cuBLAS cue blahs
392
+ cuBLAS q blahs
393
+ cuDNN c u d n n
394
+ cuDNN cue d n n
395
+ cuDNN q d n n
396
+ cuOpt cue opt
397
+ cuOpt q opt
398
+ cuQuantum cue quantum
399
+ cuQuantum q quantum
400
+ dr. doctor
401
+ e.g. for example
402
+ es3 e s three
403
+ ext4 e x t four
404
+ i3 i three
405
+ i5 i five
406
+ i7 i seven
407
+ i9 i nine
408
+ iOS ios
409
+ iTPMS i t p m s
410
+ mr. mister
411
+ mrs. misses
412
+ on TikTok on tick tock
413
+ sFTP s f t p
414
+ st. saint
415
+ x1 x one
416
+ x2 x two
417
+ x8 x eight