WeTextProcessing 1.0.0__py3-none-any.whl → 1.0.2__py3-none-any.whl

{WeTextProcessing-1.0.0.dist-info → WeTextProcessing-1.0.2.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: WeTextProcessing
-Version: 1.0.0
+Version: 1.0.2
 Summary: WeTextProcessing, including TN & ITN
 Home-page: https://github.com/wenet-e2e/WeTextProcessing
 Author: Zhendong Peng, Xingchen Song

{WeTextProcessing-1.0.0.dist-info → WeTextProcessing-1.0.2.dist-info}/RECORD

@@ -1,7 +1,7 @@
 itn/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 itn/__main__.py,sha256=jFh8bEB_lC5y7ab9VMqAxWRaI8D-SpBSTiHfMJC1b10,65
 itn/main.py,sha256=NnT9Sh3VTEI0sX_DO9IGVsXCWtZyfRbzj9Sj_6cBxqM,2610
-itn/zh_itn_tagger.fst,sha256=CtbG9WnTvHZFwtKN4Y2-1IViI5NEWLDepbSVp91LNDU,1602718
+itn/zh_itn_tagger.fst,sha256=32FSrQ6gcFYNTUYaHinnrdaMZUpAwuDmvu-EPnki8aI,1172366
 itn/zh_itn_verbalizer.fst,sha256=pHXRUfzls2UmWwTmHHhYmBWMusIfWpU3I7IwxeRjn28,119018
 itn/chinese/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 itn/chinese/inverse_normalizer.py,sha256=9GpVP7_BCBoCIpoZkPa8aq-n7M56GhY9l4mp3jUGwpM,3563
@@ -26,7 +26,7 @@ itn/chinese/data/time/minute.tsv,sha256=R2HgErTDIq_rxxRwuEotdZFmDbBgZP2DE1Ejxp6A
 itn/chinese/data/time/noon.tsv,sha256=rWP0aMoTlQ7yEUcav_wXfGLBz2sHMwT37eLmDKZ16X0,72
 itn/chinese/data/time/second.tsv,sha256=fBb-Gt3U0ehbVnM51ngS8n_eEz2bqdpLJvMY7pdeLUg,849
 itn/chinese/rules/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-itn/chinese/rules/cardinal.py,sha256=Y9T-qPfIxuiFU6UT_v8vwAMvNAtzv3ofpBd6OWEvcsg,8006
+itn/chinese/rules/cardinal.py,sha256=pFhQ934IqZH4isx7I8el2xvHtr0padu57xsZMCor8YU,7977
 itn/chinese/rules/char.py,sha256=wWnTfzJqE5j7kvEao9rKSN29OVFGNnL25MxS7PcfWEM,965
 itn/chinese/rules/date.py,sha256=8pB_S9BFKYd5rXGxzfjAcas-ecLM4dUnhNIsp_vZgI0,2447
 itn/chinese/rules/fraction.py,sha256=jGhKx91TtscqPFwDtqaI0zeMCAsCFA68KPCWBXtT030,2030
@@ -42,21 +42,21 @@ itn/chinese/test/normalizer_test.py,sha256=a9nqt-is07zYOl3XDAhjUyNzBlwBe3qdaHtvT
 itn/chinese/test/utils.py,sha256=aCZH5BL0dghGjemhVwfISBUxLFMG_ceMzwzuOb5ht28,1094
 tn/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 tn/__main__.py,sha256=OUMsgyp8SWb7raJY_py61uM29j023yE5po95O9yinzk,64
-tn/en_tn_tagger.fst,sha256=CBUjSibReC6Xgcu3Njrkxlp1HBC6Btne2yFze9Bv-Is,5248906
-tn/en_tn_verbalizer.fst,sha256=APbSGmx3ML5q5g-cSAW4mPVpOHloTqXvv015eVCVokg,1692102
+tn/en_tn_tagger.fst,sha256=gqjt_9TO5osM29IWhLzHwcu-3z-NJHqkrCs04qfmANg,5418482
+tn/en_tn_verbalizer.fst,sha256=54G4Z7G1_cIs20skJCCwq9DRPmexqExTtGYiCdnB2gw,1710934
 tn/main.py,sha256=Q45id3b2g71MPRLnaHpo8iWPc38MeyZKigNd5FinNvI,3578
-tn/processor.py,sha256=6Cdb1xqo-82BH1wxf1zMHoXnwi3Tmgo_5KvwEbP-6kM,4840
+tn/processor.py,sha256=CUmJ7B0OyweByf7U0yi2TYqBbEhhWTcqs03_0XhTosg,5012
 tn/token_parser.py,sha256=nsTYzug-gq2HYnTJHja_HXntoRVX3gITQAlI9z8WvWU,4487
 tn/utils.py,sha256=lJLuW2jkobgXCyNYl29zhtyb3B80CS7Pqz7zqHRVr4c,2792
-tn/zh_tn_tagger.fst,sha256=zZ7zLr-Y774X3lb9bnmUv9Fs3Gv-LXa44X0rMFL6a9g,3477714
+tn/zh_tn_tagger.fst,sha256=vIfJeXdWDvOSwLVmuNDeWktIq29UkcWUfbKjBnMY3cA,3479454
 tn/zh_tn_verbalizer.fst,sha256=VoZnjew7YiDRoDC6TgDHKXhrkcSGOz030Pnqad8JhcE,558366
 tn/chinese/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 tn/chinese/normalizer.py,sha256=4FiYRyx_cfzTL77gonlcpSyGxFbZbVdupFRRLx-J2UQ,3851
 tn/chinese/data/char/charset_extension.tsv,sha256=kUxQXkh5k1jeYpkNplRBlIZ6vzCUjNn23RNku5XPGvk,24
 tn/chinese/data/char/charset_national_standard_2013_8105.tsv,sha256=7GLEH98Fh_T2L_UhgD8LOQzPps7XBbT3ZM0KKK1etAQ,32616
 tn/chinese/data/char/fullwidth_to_halfwidth.tsv,sha256=Xs9kwylqrJp-Ew5Z_31686L2elA8r24q0FYKqPfNRe4,570
-tn/chinese/data/char/punctuations_zh.tsv,sha256=TpC5Ncogj0uJ980VRkpYq3HsxqV_MENsY-eLTRkkMTU,288
-tn/chinese/data/char/traditional_to_simple.tsv,sha256=hqE4gU1wmdhhWBPDhmz5Oy-kJdJCxDnEYWefCkEezzQ,25160
+tn/chinese/data/char/punctuations_zh.tsv,sha256=8X7rzt0P4AWyp9EWA_xkgwFf1dk8Z-ME6FTuVoSh9qw,292
+tn/chinese/data/char/traditional_to_simple.tsv,sha256=0FXZVJjOtSpGpzkcc1KcQRxnBRtPcWVrqgzAYKGXLPw,25168
 tn/chinese/data/date/d.tsv,sha256=GrIfCvNeZho4jZssTbtzJoecE5HEt7PG4t_WdWfJvdc,394
 tn/chinese/data/date/dd.tsv,sha256=oPLQNhJ08cOUtqI_uM1aQ8by4kOv4zkmd-kux2kMb_Q,403
 tn/chinese/data/date/m.tsv,sha256=KELO_dRSedVfNfqsjhPJRrh8SofOYKcPcpi3a-DsZBc,117
@@ -110,7 +110,7 @@ tn/chinese/test/token_parser_test.py,sha256=8JcSqlvaYTZ46rZgMsfAVNmcYez6t0gonhnx
 tn/chinese/test/utils.py,sha256=aCZH5BL0dghGjemhVwfISBUxLFMG_ceMzwzuOb5ht28,1094
 tn/chinese/test/whitelist_test.py,sha256=oPB-7_g8l6zhAeH5fp7NlaRJYcDNz7VAOKwhCnebGdc,1003
 tn/english/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-tn/english/normalizer.py,sha256=ABmm9a7mwmowXtj-ad4erT2VtpHVzkIlbwwBTCV7CEw,3940
+tn/english/normalizer.py,sha256=YNd9PYib_3m9ExMmxd0SAq0CWpEHxpEi4S0ZpNrqyVs,3921
 tn/english/data/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 tn/english/data/suppletive.tsv,sha256=iih47clDISe35FR-p4Usu_coR3oQy1miXd4VsBJSsyA,1843
 tn/english/data/address/__init__.py,sha256=EqFAHHjK7kJqR_FV8vBLddBJWhbVD8pREEfm8C5nSEE,623
@@ -121,6 +121,7 @@ tn/english/data/date/day.tsv,sha256=JxNevqMlx8dz3ogpFDbU3QUuHRHnfUJYx5ypBbZrKAc,
 tn/english/data/date/month_abbr.tsv,sha256=daz29xDyEiqRizcofy1b4Vbhn2OPNKADILnRLXX2sRs,423
 tn/english/data/date/month_name.tsv,sha256=7DOgcRmfKqDLPaggBaZeCKcF8X1W23Y2A0VLwzv3ggc,500
 tn/english/data/date/month_number.tsv,sha256=FNZPT3VwSFefhsOxLHzKrgKR4qWIytCgdhQ3PeVkwa8,234
+tn/english/data/date/week.tsv,sha256=zJAjMh4qItucLFB7aesJJ7owJ8Vj2FfanYGX8rE0LVw,233
 tn/english/data/date/year_suffix.tsv,sha256=5Ph-HOHLY5p1vVJ2IEGMWnmO1zGWIPBbF9WCpzhZ9Zk,128
 tn/english/data/electronic/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 tn/english/data/electronic/domain.tsv,sha256=BOe1iCXxqWc6F8rSpAdPB3k2ujV1g3jbEVFNCQVXAXQ,142
@@ -162,14 +163,14 @@ tn/english/data/time/__init__.py,sha256=TS5NMVLZXlTIgbt7grYmlxBfRKrWELPnT9DedX9k
 tn/english/data/time/suffix.tsv,sha256=VX7aFlrHb3NEaJpdpZseH6i1yb0CBorBTrO4roELk0g,84
 tn/english/data/time/zone.tsv,sha256=OEU5PU0XS3hQS9Vsgcn-ADv_0PMok6uyirIf9-SeL-I,116
 tn/english/data/whitelist/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-tn/english/data/whitelist/alternatives.tsv,sha256=NQYfs_PDJo9WCSK5wtJoMkkxJpW2pOZTiHvTy475uFQ,306
+tn/english/data/whitelist/alternatives.tsv,sha256=MGekzXHOfp8xfPH85agBlAvtBFZxYNDMaaGZE-QhrG4,318
 tn/english/data/whitelist/alternatives_all_format.tsv,sha256=VzUfxtM9-_dVcM6JMSl7tSGtSZsoljAsKzzTxPeEuBI,155
 tn/english/data/whitelist/asr.tsv,sha256=b3J6hOJifqH_DwwZ4BOOVEQ45Q43B3PW2fZEztfdjpo,175146
 tn/english/data/whitelist/asr_with_pc.tsv,sha256=wqeEeG4O742D2vroywzbstwmr-s6lGHVv5KAD_K-qiU,865
 tn/english/data/whitelist/ipa_symbols.tsv,sha256=ChsK_JiztbsqNnFYNQ0xZ9QCFUxeTqRzm8vCkydQvl8,2532
 tn/english/data/whitelist/lj_speech.tsv,sha256=-sZeHERreSBZGbLiJh0q_laYvXWG5mvMAUuXkYiHQdI,263
 tn/english/data/whitelist/symbol.tsv,sha256=rIyi-7u3X-DoBslNDaAQghNVWlD6yLAgYS2nuGKQUJ4,240
-tn/english/data/whitelist/tts.tsv,sha256=6tRaxMf2D4YspkWKde4Mdx1NdE3ZzQTk0Bh-nOWlsLA,33630
+tn/english/data/whitelist/tts.tsv,sha256=JZzUz_qoLQbqkfLsluDklCIeqBHNh2nQgAWcAea03Y0,33073
 tn/english/rules/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 tn/english/rules/cardinal.py,sha256=w3HvKbq86mZjb3MhXc96D_Ajaem1raCHs0pCGvhQaRs,8424
 tn/english/rules/date.py,sha256=QoUczkPwAeYanT41_8Qpi_FqvuSfvXX5LHdEgcsLkVA,13683
@@ -179,8 +180,8 @@ tn/english/rules/fraction.py,sha256=ltGhbd1Qeb-BsuVCPYq8cc__k0Xkkn4WBUiz6dFs8J8,
 tn/english/rules/measure.py,sha256=DjQnq_9UXL-pHdqaHkp3oUv4qEhiGB_764YmUEq9v6w,14025
 tn/english/rules/money.py,sha256=0hjQWUdI4wWsOVKkkBs5lvFsQaMIWOQcs6oOLBhQzDM,5457
 tn/english/rules/ordinal.py,sha256=xFUaNvPx2rKDJz3TMiE2RKje6gcvcVImr3sXixWfOKU,3989
-tn/english/rules/punctuation.py,sha256=KyypaQs7SsQC1RCv-h4NGu4Jm-PvuSGUyM3mtPbCMws,2979
-tn/english/rules/range.py,sha256=Ulq6Jpg_vHkcpmSDyhKZWHxVr5MXzEdeRChidlH4Rik,5443
+tn/english/rules/punctuation.py,sha256=he6GQ-6CUf9lMzPDCU2PW_RqboQHwhb2V5M5mdBdFHo,3107
+tn/english/rules/range.py,sha256=lEjLyFBJSCHnHlQBdrzhiUdNVIiwTQClPz4DNRqHnB4,5730
 tn/english/rules/roman.py,sha256=pQFuSIjzWnMN-jiSuVqv3AxJEA9kA8s0NUgGpCCVbiU,6051
 tn/english/rules/telephone.py,sha256=JkhHZU1O2h4a_W3QHVBY1lOQvq4Tys5JwAlL1Kkt8Pg,8275
 tn/english/rules/time.py,sha256=7quSQrLifeJ4WLwAyDkuvGe1-lNomN_MjvFAP3zHBas,7764
@@ -203,9 +204,9 @@ tn/english/test/time_test.py,sha256=4E_PS-D5caTUZgrI-UL7q0aoqmtD8np7yeXUkGyc1Qo,
 tn/english/test/utils.py,sha256=aCZH5BL0dghGjemhVwfISBUxLFMG_ceMzwzuOb5ht28,1094
 tn/english/test/whitelist_test.py,sha256=fnlpnwiDgQWsl5ONZklWXEyLQoUSuwhdowcV7a8fXvw,1023
 tn/english/test/word_test.py,sha256=ACguloXPl4Ttw8fj9F_wvr1coaQAoj_MLOJ74wyy-WY,954
-WeTextProcessing-1.0.0.dist-info/LICENSE,sha256=xx0jnfkXJvxRnG63LTGOxlggYnIysveWIZ6H3PNdCrQ,11357
-WeTextProcessing-1.0.0.dist-info/METADATA,sha256=W2xwGzXAdYq7A-SD8NBsGPbjytUXUiNQNnXt5r2qCqA,5178
-WeTextProcessing-1.0.0.dist-info/WHEEL,sha256=GJ7t_kWBFywbagK5eo9IoUwLW6oyOeTKmQ-9iHFVNxQ,92
-WeTextProcessing-1.0.0.dist-info/entry_points.txt,sha256=iAd8tFSNA8AY5fUECPlk9KaDFC3ISvDjoKAZVsCWPjE,61
-WeTextProcessing-1.0.0.dist-info/top_level.txt,sha256=6NasE_UheQezX9_0V3MbiOSnTQpvM_funMjv4wpMGgA,7
-WeTextProcessing-1.0.0.dist-info/RECORD,,
+WeTextProcessing-1.0.2.dist-info/LICENSE,sha256=xx0jnfkXJvxRnG63LTGOxlggYnIysveWIZ6H3PNdCrQ,11357
+WeTextProcessing-1.0.2.dist-info/METADATA,sha256=vINHY2ibaBLdN56EpTFgkk5Wn-jde8RfYy1gS6qnn7A,5178
+WeTextProcessing-1.0.2.dist-info/WHEEL,sha256=GJ7t_kWBFywbagK5eo9IoUwLW6oyOeTKmQ-9iHFVNxQ,92
+WeTextProcessing-1.0.2.dist-info/entry_points.txt,sha256=iAd8tFSNA8AY5fUECPlk9KaDFC3ISvDjoKAZVsCWPjE,61
+WeTextProcessing-1.0.2.dist-info/top_level.txt,sha256=6NasE_UheQezX9_0V3MbiOSnTQpvM_funMjv4wpMGgA,7
+WeTextProcessing-1.0.2.dist-info/RECORD,,

itn/chinese/rules/cardinal.py

@@ -69,7 +69,7 @@ class Cardinal(Processor):
                                           | add_weight(addzero**2, 1.0)))
         # 一千一百一十一 => 1111, 一千零一十一 => 1011, 一千零一 => 1001
         # 一千一 => 1100, 一千 => 1000
-        thousand = ((hundred | teen | tens | digits) + delete('千') +
+        thousand = (digit + delete('千') +
                     (hundred
                      | add_weight(zero + (tens | teen), 0.1)
                      | add_weight(addzero + zero + digit, 0.5)
@@ -78,7 +78,7 @@ class Cardinal(Processor):
         # 10001111, 1001111, 101111, 11111, 10111, 10011, 10001, 10000
         if self.enable_million:
             ten_thousand = (
-                (thousand | hundred | teen | tens | digits) + delete('万') +
+                (thousand | hundred | teen | tens | digit) + delete('万') +
                 (thousand
                  | add_weight(zero + hundred, 0.1)
                  | add_weight(addzero + zero + (tens | teen), 0.5)
@@ -87,7 +87,7 @@ class Cardinal(Processor):
                  | add_weight(addzero**4, 1.0)))
         else:
             ten_thousand = (
-                (teen | tens | digits) + delete('万') +
+                (teen | tens | digit) + delete('万') +
                 (thousand
                  | add_weight(zero + hundred, 0.1)
                  | add_weight(addzero + zero + (tens | teen), 0.5)

tn/chinese/data/char/punctuations_zh.tsv

@@ -12,6 +12,7 @@
 、
 。
 〃
+《
 》
 「
 」

tn/chinese/data/char/traditional_to_simple.tsv

@@ -3143,3 +3143,4 @@
 龔	龚
 龕	龛
 龜	龟
+後	后

tn/english/data/date/week.tsv

@@ -0,0 +1,18 @@
+Mon	Monday
+Mon.	Monday
+Tu	Tuesday
+Tu.	Tuesday
+Wed	Wednesday
+Wed.	Wednesday
+Th	Thursday
+Th.	Thursday
+Thur	Thursday
+Thur.	Thursday
+Thurs	Thursday
+Thurs.	Thursday
+Fri	Friday
+Fri.	Friday
+Sat	Saturday
+Sat.	Saturday
+Sun	Sunday
+Sun.	Sunday

tn/english/data/whitelist/alternatives.tsv

@@ -23,4 +23,5 @@ NO.	number
 NO.	number
 No.	number
 VOL.	Volume
+Vol.	Volume
 TV	Television

tn/english/data/whitelist/tts.tsv

@@ -3,24 +3,13 @@ Hon.	honorable
 Mt.	Mount
 Maj.	Major
 Rev.	Reverend
-w/o	without
-a/c number	account number
-c/o	care of
 Gov.	governor
-vs.	versus
-dept.	department
-prof.	professor
-est.	estimated
-vol.	volume
-bldg.	building
 Bldg.	Building
-apt.	apartment
 Apt.	Apartment
 World War II.	World War two
 World War II	World War two
 World War I.	World War one
 World War I	World War one
-etc.	etcetera.
 SnO2	tin four oxide
 HVAC	H-vac
 SPDR	spider
@@ -77,24 +66,14 @@ ZIP	zip
 Ψ	psi
 ζ	zeta
 Ζ	zeta
-ltd	limited
-int'l	international
 A. D	AD
 A.D	AD
-a. d	AD
-a.d	AD
-a. d.	AD
-a.d.	AD
 B. C	BC
 B.C	BC
-b. c	BC
-b.c	BC
 A. D.	AD
 A.D.	AD
 B. C.	BC
 B.C.	BC
-b. c.	BC
-b.c.	BC
 A. A.	a a
 A.A.	AA
 A&A	A and A
@@ -222,8 +201,6 @@ Ajok	AJOK
 Ajsa	AJSA
 Aju	AJU
 Akaa	AKAA
-a.k.a.	AKA
-aka.	AKA
 Aka.	AKA
 Aka	AKA
 A&K	A and K
@@ -240,8 +217,6 @@ Alh	ALH
 ALUs	ALU's
 A'ma	AMA
 A&M	A and M
-a.m.	AM
-a.m	AM
 A. M.	AM
 A.M.	AM
 AMAs	AMA's
@@ -414,7 +389,6 @@ Ba'al	BAAL
 Baal	BAAL
 Baam	BAAM
 Baap	BAAP
-ba'as	BAA's
 Baat	BAAT
 Ba'ath	BAATH
 B. A.	BA
@@ -567,7 +541,6 @@ CBGBs	CBGB's
 CbiXS	CBIXS
 CB&Q	CB and Q
 CBs	CB's
-'c	C
 CcaA	CCAA
 Cca	CCA
 C&C	C and C
@@ -595,7 +568,6 @@ CdTe	CDTE
 CDTi	CDTI
 CD&V	CD and V
 Cec	CEC
-c'e	CE
 C'e	CE
 Ce	CE
 C. E.	CE
@@ -721,7 +693,6 @@ Ctrip	CTRIP
 C. U.	CU
 Cuyp	CUYP
 C&V	C and V
-'cv	CV
 C. V.	CV
 C.V.	CV
 CVEs	CVE's
@@ -875,7 +846,6 @@ DSGi	DSGI
 DSi	DSI
 DSMs	DSM's
 D&SNGRR	D and SNGRR
-dsu'	DSU
 DTCs	DTC's
 DTDs	DTD's
 D. T.	DT
@@ -949,8 +919,6 @@ eFS	EFS
 Efs	EF's
 Efu	EFU
 Egba	EGBA
-e.g.	for  example
-e. g.	for  example
 EG&G	EG and G
 Egi	EGI
 E&GR	E and GR
@@ -1020,7 +988,6 @@ Eph	EPH
 Ephs	EPH's
 Epi	EPI
 ePO	EPO
-epos'	EPO's
 Epos	EPO's
 E.P.s	EP's
 Eq	EQ
@@ -1455,7 +1422,6 @@ Idm	IDM
 IDPs	IDP's
 IDs	ID's
 IDx	IDX
-i.e.	that  is
 Ifa	IFA
 IfF	IFF
 Ifft	IFFT
@@ -1469,7 +1435,6 @@ IFRSs	IFRS's
 Iga	IGA
 IgA	IGA
 IGFs	IGF's
-'ig	IG
 Ig	IG
 I.G.	IG
 IgM	IGM
@@ -1535,7 +1500,6 @@ INTs	INT's
 Intu	INTU
 Ioba	IOBA
 IODs	IOD's
-io.	IO
 .I.o.	IO
 Io.	IO
 Io	IO
@@ -1569,7 +1533,6 @@ I.Q.	IQ
 Irgm	IRGM
 Irig	IRIG
 Iril	IRIL
-i'r	IR
 Ir	IR
 I. R.	IR
 I.R.	IR
@@ -1680,7 +1643,6 @@ J&R	J and R
 J. R.	JR
 J.R.	JR
 Jr.	junior
-jr.	junior
 Jr	junior
 J.R.'s	JR's
 Js	J's
@@ -1747,7 +1709,6 @@ K.M.	KM
 K&N	K and N
 K. N.	KN
 K.N.	KN
-k'o	KO
 K. O.	KO
 K.O.	KO
 KPIs	KPI's
@@ -2210,8 +2171,6 @@ PLCs	PLC's
 P. L.	PL
 P.L.	PL
 PMMoV	PMMOV
-p.m.	PM
-p.m	PM
 P. M.	PM
 P.M.	PM
 PMs	PM's
@@ -2437,7 +2396,6 @@ S.E.S.'s	SES's
 Seyh	SEYH
 Sfax	SFAX
 SFFCo	SFFCO
-'sf	SF
 S. F.	SF
 S.F.	SF
 S.F.'s	SF's
@@ -2603,7 +2561,6 @@ Tiv	TIV
 Tiy's	TIY's
 T. J.	TJ
 T.J.	TJ
-t'ju	TJU
 Tko	TKO
 T. K.	TK
 T.K.	TK
@@ -2746,7 +2703,6 @@ Ukoh	UKOH
 Uko	UKO
 Ukr	UKR
 Ukui	UKUI
-u'k	UK
 Uk	UK
 U.K.	UK
 Ukwu	UKWU
@@ -2923,8 +2879,6 @@ VSANs	VSAN's
 VSATs	VSAT's
 Vsekh	VSEKH
 VSPs	VSP's
-vs.	versus
-_vs._	versus
 VTE	VT eL
 Vtic	VTIC
 VTi	VTI
@@ -2941,7 +2895,6 @@ V. Z.	VZ
 Waay	WAAY
 Wa'il	WAIL
 Wakf	WAKF
-wa'l	WAL
 Waqt	WAQT
 Wasl	WASL
 Watfa	WATFA

tn/english/normalizer.py

@@ -63,7 +63,6 @@ class Normalizer(Processor):
              | time | measure | money
              | telephone | electronic
              | whitelist
-             | punct
              | rang).optimize() + (punct.plus | self.DELETE_SPACE)
         # delete the last space
         self.tagger = tagger.star @ self.build_rule(delete(' '), r='[EOS]')
@@ -92,6 +91,6 @@ class Normalizer(Processor):
              | electronic
              | whitelist
              | punct
-             | rang).optimize() + punct.ques + self.INSERT_SPACE
+             | rang).optimize() + (punct.plus | self.INSERT_SPACE)
         self.verbalizer = verbalizer.star @ self.build_rule(delete(' '),
                                                             r='[EOS]')

tn/english/rules/punctuation.py

@@ -17,7 +17,7 @@ from unicodedata import category
 
 from pynini.examples import plurals
 from pynini import cross, union, closure, accep
-from pynini.lib.pynutil import delete, insert
+from pynini.lib.pynutil import add_weight, delete, insert
 
 from tn.processor import Processor
 from tn.utils import get_abs_path, load_labels
@@ -71,11 +71,14 @@ class Punctuation(Processor):
         punct = plurals._priority_union(emphasis, punct, closure(self.VCHAR))
 
         self.graph = punct
-        final_graph = insert("v: \"") + punct + insert("\"")
+        final_graph = insert("v: \"") + add_weight(
+            accep(" "), -1.0).star + punct + add_weight(
+                accep(" "), -1.0).star + insert("\"")
         self.tagger = self.add_tokens(final_graph)
 
     def build_verbalizer(self):
-        punct = closure(self.punct | cross('\\\\\\', '\\') | cross('\\"', '"'),
-                        1)
+        punct = closure(
+            self.punct | cross('\\\\\\', '\\') | cross('\\"', '"')
+            | accep(" "), 1)
         verbalizer = delete('v: "') + punct + delete('"')
         self.verbalizer = self.delete_tokens(verbalizer)

tn/english/rules/range.py

@@ -17,6 +17,7 @@ import pynini
 from pynini.lib import pynutil
 
 from tn.processor import Processor
+from tn.utils import get_abs_path
 from tn.english.rules.cardinal import Cardinal
 from tn.english.rules.time import Time
 from tn.english.rules.date import Date
@@ -45,14 +46,19 @@ class Range(Processor):
         time = time.tagger @ time.verbalizer
         date = Date(deterministic=self.deterministic)
         date = date.tagger @ date.verbalizer
+        week = pynini.string_file(get_abs_path("english/data/date/week.tsv"))
         delete_space = pynini.closure(pynutil.delete(" "), 0, 1)
 
         approx = pynini.cross("~", "approximately")
 
+        # WEEK
+        week_graph = week + delete_space + (pynini.cross("-", " to ")
+                                            | approx) + delete_space + week
+
         # TIME
         time_graph = time + delete_space + pynini.cross(
             "-", " to ") + delete_space + time
-        self.graph = time_graph | (approx + time)
+        self.graph = time_graph | (approx + time) | week_graph
 
         # YEAR
         date_year_four_digit = (self.DIGIT**4 +

tn/processor.py

@@ -23,9 +23,6 @@ from pynini import (cdrewrite, cross, difference, escape, Fst, shortestpath,
 from pynini.lib import byte, utf8
 from pynini.lib.pynutil import delete, insert
 
-logging.basicConfig(level=logging.DEBUG,
-                    format='%(asctime)s WETEXT %(levelname)s %(message)s')
-
 
 class Processor:
 
@@ -78,6 +75,13 @@ class Processor:
         self.verbalizer = self.delete_tokens(verbalizer)
 
     def build_fst(self, prefix, cache_dir, overwrite_cache):
+        logger = logging.getLogger('wetext-{}'.format(self.name))
+        logger.setLevel(logging.INFO)
+        handler = logging.StreamHandler()
+        fmt = logging.Formatter('%(asctime)s WETEXT %(levelname)s %(message)s')
+        handler.setFormatter(fmt)
+        logger.addHandler(handler)
+
         os.makedirs(cache_dir, exist_ok=True)
         tagger_name = '{}_tagger.fst'.format(prefix)
         verbalizer_name = '{}_verbalizer.fst'.format(prefix)
@@ -88,20 +92,20 @@ class Processor:
         exists = os.path.exists(tagger_path) and os.path.exists(
             verbalizer_path)
         if exists and not overwrite_cache:
-            logging.info("found existing fst: {}".format(tagger_path))
-            logging.info("                    {}".format(verbalizer_path))
-            logging.info("skip building fst for {} ...".format(self.name))
+            logger.info("found existing fst: {}".format(tagger_path))
+            logger.info("                    {}".format(verbalizer_path))
+            logger.info("skip building fst for {} ...".format(self.name))
             self.tagger = Fst.read(tagger_path).optimize()
             self.verbalizer = Fst.read(verbalizer_path).optimize()
         else:
-            logging.info("building fst for {} ...".format(self.name))
+            logger.info("building fst for {} ...".format(self.name))
             self.build_tagger()
             self.build_verbalizer()
             self.tagger.optimize().write(tagger_path)
             self.verbalizer.optimize().write(verbalizer_path)
-            logging.info("done")
-            logging.info("fst path: {}".format(tagger_path))
-            logging.info("          {}".format(verbalizer_path))
+            logger.info("done")
+            logger.info("fst path: {}".format(tagger_path))
+            logger.info("          {}".format(verbalizer_path))
 
     def tag(self, input):
         if len(input) == 0: