WeTextProcessing 1.0.0__tar.gz → 1.0.2__tar.gz

{WeTextProcessing-1.0.0 → WeTextProcessing-1.0.2}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: WeTextProcessing
-Version: 1.0.0
+Version: 1.0.2
 Summary: WeTextProcessing, including TN & ITN
 Home-page: https://github.com/wenet-e2e/WeTextProcessing
 Author: Zhendong Peng, Xingchen Song

{WeTextProcessing-1.0.0 → WeTextProcessing-1.0.2}/WeTextProcessing.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: WeTextProcessing
-Version: 1.0.0
+Version: 1.0.2
 Summary: WeTextProcessing, including TN & ITN
 Home-page: https://github.com/wenet-e2e/WeTextProcessing
 Author: Zhendong Peng, Xingchen Song

{WeTextProcessing-1.0.0 → WeTextProcessing-1.0.2}/WeTextProcessing.egg-info/SOURCES.txt

@@ -130,6 +130,7 @@ tn/english/data/date/day.tsv
 tn/english/data/date/month_abbr.tsv
 tn/english/data/date/month_name.tsv
 tn/english/data/date/month_number.tsv
+tn/english/data/date/week.tsv
 tn/english/data/date/year_suffix.tsv
 tn/english/data/electronic/__init__.py
 tn/english/data/electronic/domain.tsv

{WeTextProcessing-1.0.0 → WeTextProcessing-1.0.2}/itn/chinese/rules/cardinal.py

@@ -69,7 +69,7 @@ class Cardinal(Processor):
                                           | add_weight(addzero**2, 1.0)))
         # 一千一百一十一 => 1111, 一千零一十一 => 1011, 一千零一 => 1001
         # 一千一 => 1100, 一千 => 1000
-        thousand = ((hundred | teen | tens | digits) + delete('千') +
+        thousand = (digit + delete('千') +
                     (hundred
                      | add_weight(zero + (tens | teen), 0.1)
                      | add_weight(addzero + zero + digit, 0.5)
@@ -78,7 +78,7 @@ class Cardinal(Processor):
         # 10001111, 1001111, 101111, 11111, 10111, 10011, 10001, 10000
         if self.enable_million:
             ten_thousand = (
-                (thousand | hundred | teen | tens | digits) + delete('万') +
+                (thousand | hundred | teen | tens | digit) + delete('万') +
                 (thousand
                  | add_weight(zero + hundred, 0.1)
                  | add_weight(addzero + zero + (tens | teen), 0.5)
@@ -87,7 +87,7 @@ class Cardinal(Processor):
                  | add_weight(addzero**4, 1.0)))
         else:
             ten_thousand = (
-                (teen | tens | digits) + delete('万') +
+                (teen | tens | digit) + delete('万') +
                 (thousand
                  | add_weight(zero + hundred, 0.1)
                  | add_weight(addzero + zero + (tens | teen), 0.5)

{WeTextProcessing-1.0.0 → WeTextProcessing-1.0.2}/tn/chinese/data/char/punctuations_zh.tsv

@@ -12,6 +12,7 @@
 、
 。
 〃
+《
 》
 「
 」

{WeTextProcessing-1.0.0 → WeTextProcessing-1.0.2}/tn/chinese/data/char/traditional_to_simple.tsv

@@ -3143,3 +3143,4 @@
 龔	龚
 龕	龛
 龜	龟
+後	后

WeTextProcessing-1.0.2/tn/english/data/date/week.tsv

@@ -0,0 +1,18 @@
+Mon	Monday
+Mon.	Monday
+Tu	Tuesday
+Tu.	Tuesday
+Wed	Wednesday
+Wed.	Wednesday
+Th	Thursday
+Th.	Thursday
+Thur	Thursday
+Thur.	Thursday
+Thurs	Thursday
+Thurs.	Thursday
+Fri	Friday
+Fri.	Friday
+Sat	Saturday
+Sat.	Saturday
+Sun	Sunday
+Sun.	Sunday

{WeTextProcessing-1.0.0 → WeTextProcessing-1.0.2}/tn/english/data/whitelist/alternatives.tsv

@@ -23,4 +23,5 @@ NO.	number
 NO.	number
 No.	number
 VOL.	Volume
+Vol.	Volume
 TV	Television

{WeTextProcessing-1.0.0 → WeTextProcessing-1.0.2}/tn/english/data/whitelist/tts.tsv

@@ -3,24 +3,13 @@ Hon.	honorable
 Mt.	Mount
 Maj.	Major
 Rev.	Reverend
-w/o	without
-a/c number	account number
-c/o	care of
 Gov.	governor
-vs.	versus
-dept.	department
-prof.	professor
-est.	estimated
-vol.	volume
-bldg.	building
 Bldg.	Building
-apt.	apartment
 Apt.	Apartment
 World War II.	World War two
 World War II	World War two
 World War I.	World War one
 World War I	World War one
-etc.	etcetera.
 SnO2	tin four oxide
 HVAC	H-vac
 SPDR	spider
@@ -77,24 +66,14 @@ ZIP	zip
 Ψ	psi
 ζ	zeta
 Ζ	zeta
-ltd	limited
-int'l	international
 A. D	AD
 A.D	AD
-a. d	AD
-a.d	AD
-a. d.	AD
-a.d.	AD
 B. C	BC
 B.C	BC
-b. c	BC
-b.c	BC
 A. D.	AD
 A.D.	AD
 B. C.	BC
 B.C.	BC
-b. c.	BC
-b.c.	BC
 A. A.	a a
 A.A.	AA
 A&A	A and A
@@ -222,8 +201,6 @@ Ajok	AJOK
 Ajsa	AJSA
 Aju	AJU
 Akaa	AKAA
-a.k.a.	AKA
-aka.	AKA
 Aka.	AKA
 Aka	AKA
 A&K	A and K
@@ -240,8 +217,6 @@ Alh	ALH
 ALUs	ALU's
 A'ma	AMA
 A&M	A and M
-a.m.	AM
-a.m	AM
 A. M.	AM
 A.M.	AM
 AMAs	AMA's
@@ -414,7 +389,6 @@ Ba'al	BAAL
 Baal	BAAL
 Baam	BAAM
 Baap	BAAP
-ba'as	BAA's
 Baat	BAAT
 Ba'ath	BAATH
 B. A.	BA
@@ -567,7 +541,6 @@ CBGBs	CBGB's
 CbiXS	CBIXS
 CB&Q	CB and Q
 CBs	CB's
-'c	C
 CcaA	CCAA
 Cca	CCA
 C&C	C and C
@@ -595,7 +568,6 @@ CdTe	CDTE
 CDTi	CDTI
 CD&V	CD and V
 Cec	CEC
-c'e	CE
 C'e	CE
 Ce	CE
 C. E.	CE
@@ -721,7 +693,6 @@ Ctrip	CTRIP
 C. U.	CU
 Cuyp	CUYP
 C&V	C and V
-'cv	CV
 C. V.	CV
 C.V.	CV
 CVEs	CVE's
@@ -875,7 +846,6 @@ DSGi	DSGI
 DSi	DSI
 DSMs	DSM's
 D&SNGRR	D and SNGRR
-dsu'	DSU
 DTCs	DTC's
 DTDs	DTD's
 D. T.	DT
@@ -949,8 +919,6 @@ eFS	EFS
 Efs	EF's
 Efu	EFU
 Egba	EGBA
-e.g.	for  example
-e. g.	for  example
 EG&G	EG and G
 Egi	EGI
 E&GR	E and GR
@@ -1020,7 +988,6 @@ Eph	EPH
 Ephs	EPH's
 Epi	EPI
 ePO	EPO
-epos'	EPO's
 Epos	EPO's
 E.P.s	EP's
 Eq	EQ
@@ -1455,7 +1422,6 @@ Idm	IDM
 IDPs	IDP's
 IDs	ID's
 IDx	IDX
-i.e.	that  is
 Ifa	IFA
 IfF	IFF
 Ifft	IFFT
@@ -1469,7 +1435,6 @@ IFRSs	IFRS's
 Iga	IGA
 IgA	IGA
 IGFs	IGF's
-'ig	IG
 Ig	IG
 I.G.	IG
 IgM	IGM
@@ -1535,7 +1500,6 @@ INTs	INT's
 Intu	INTU
 Ioba	IOBA
 IODs	IOD's
-io.	IO
 .I.o.	IO
 Io.	IO
 Io	IO
@@ -1569,7 +1533,6 @@ I.Q.	IQ
 Irgm	IRGM
 Irig	IRIG
 Iril	IRIL
-i'r	IR
 Ir	IR
 I. R.	IR
 I.R.	IR
@@ -1680,7 +1643,6 @@ J&R	J and R
 J. R.	JR
 J.R.	JR
 Jr.	junior
-jr.	junior
 Jr	junior
 J.R.'s	JR's
 Js	J's
@@ -1747,7 +1709,6 @@ K.M.	KM
 K&N	K and N
 K. N.	KN
 K.N.	KN
-k'o	KO
 K. O.	KO
 K.O.	KO
 KPIs	KPI's
@@ -2210,8 +2171,6 @@ PLCs	PLC's
 P. L.	PL
 P.L.	PL
 PMMoV	PMMOV
-p.m.	PM
-p.m	PM
 P. M.	PM
 P.M.	PM
 PMs	PM's
@@ -2437,7 +2396,6 @@ S.E.S.'s	SES's
 Seyh	SEYH
 Sfax	SFAX
 SFFCo	SFFCO
-'sf	SF
 S. F.	SF
 S.F.	SF
 S.F.'s	SF's
@@ -2603,7 +2561,6 @@ Tiv	TIV
 Tiy's	TIY's
 T. J.	TJ
 T.J.	TJ
-t'ju	TJU
 Tko	TKO
 T. K.	TK
 T.K.	TK
@@ -2746,7 +2703,6 @@ Ukoh	UKOH
 Uko	UKO
 Ukr	UKR
 Ukui	UKUI
-u'k	UK
 Uk	UK
 U.K.	UK
 Ukwu	UKWU
@@ -2923,8 +2879,6 @@ VSANs	VSAN's
 VSATs	VSAT's
 Vsekh	VSEKH
 VSPs	VSP's
-vs.	versus
-_vs._	versus
 VTE	VT eL
 Vtic	VTIC
 VTi	VTI
@@ -2941,7 +2895,6 @@ V. Z.	VZ
 Waay	WAAY
 Wa'il	WAIL
 Wakf	WAKF
-wa'l	WAL
 Waqt	WAQT
 Wasl	WASL
 Watfa	WATFA

{WeTextProcessing-1.0.0 → WeTextProcessing-1.0.2}/tn/english/normalizer.py

@@ -63,7 +63,6 @@ class Normalizer(Processor):
              | time | measure | money
              | telephone | electronic
              | whitelist
-             | punct
              | rang).optimize() + (punct.plus | self.DELETE_SPACE)
         # delete the last space
         self.tagger = tagger.star @ self.build_rule(delete(' '), r='[EOS]')
@@ -92,6 +91,6 @@ class Normalizer(Processor):
              | electronic
              | whitelist
              | punct
-             | rang).optimize() + punct.ques + self.INSERT_SPACE
+             | rang).optimize() + (punct.plus | self.INSERT_SPACE)
         self.verbalizer = verbalizer.star @ self.build_rule(delete(' '),
                                                             r='[EOS]')

{WeTextProcessing-1.0.0 → WeTextProcessing-1.0.2}/tn/english/rules/punctuation.py

@@ -17,7 +17,7 @@ from unicodedata import category
 
 from pynini.examples import plurals
 from pynini import cross, union, closure, accep
-from pynini.lib.pynutil import delete, insert
+from pynini.lib.pynutil import add_weight, delete, insert
 
 from tn.processor import Processor
 from tn.utils import get_abs_path, load_labels
@@ -71,11 +71,14 @@ class Punctuation(Processor):
         punct = plurals._priority_union(emphasis, punct, closure(self.VCHAR))
 
         self.graph = punct
-        final_graph = insert("v: \"") + punct + insert("\"")
+        final_graph = insert("v: \"") + add_weight(
+            accep(" "), -1.0).star + punct + add_weight(
+                accep(" "), -1.0).star + insert("\"")
         self.tagger = self.add_tokens(final_graph)
 
     def build_verbalizer(self):
-        punct = closure(self.punct | cross('\\\\\\', '\\') | cross('\\"', '"'),
-                        1)
+        punct = closure(
+            self.punct | cross('\\\\\\', '\\') | cross('\\"', '"')
+            | accep(" "), 1)
         verbalizer = delete('v: "') + punct + delete('"')
         self.verbalizer = self.delete_tokens(verbalizer)

{WeTextProcessing-1.0.0 → WeTextProcessing-1.0.2}/tn/english/rules/range.py

@@ -17,6 +17,7 @@ import pynini
 from pynini.lib import pynutil
 
 from tn.processor import Processor
+from tn.utils import get_abs_path
 from tn.english.rules.cardinal import Cardinal
 from tn.english.rules.time import Time
 from tn.english.rules.date import Date
@@ -45,14 +46,19 @@ class Range(Processor):
         time = time.tagger @ time.verbalizer
         date = Date(deterministic=self.deterministic)
         date = date.tagger @ date.verbalizer
+        week = pynini.string_file(get_abs_path("english/data/date/week.tsv"))
         delete_space = pynini.closure(pynutil.delete(" "), 0, 1)
 
         approx = pynini.cross("~", "approximately")
 
+        # WEEK
+        week_graph = week + delete_space + (pynini.cross("-", " to ")
+                                            | approx) + delete_space + week
+
         # TIME
         time_graph = time + delete_space + pynini.cross(
             "-", " to ") + delete_space + time
-        self.graph = time_graph | (approx + time)
+        self.graph = time_graph | (approx + time) | week_graph
 
         # YEAR
         date_year_four_digit = (self.DIGIT**4 +

{WeTextProcessing-1.0.0 → WeTextProcessing-1.0.2}/tn/processor.py

@@ -23,9 +23,6 @@ from pynini import (cdrewrite, cross, difference, escape, Fst, shortestpath,
 from pynini.lib import byte, utf8
 from pynini.lib.pynutil import delete, insert
 
-logging.basicConfig(level=logging.DEBUG,
-                    format='%(asctime)s WETEXT %(levelname)s %(message)s')
-
 
 class Processor:
 
@@ -78,6 +75,13 @@ class Processor:
         self.verbalizer = self.delete_tokens(verbalizer)
 
     def build_fst(self, prefix, cache_dir, overwrite_cache):
+        logger = logging.getLogger('wetext-{}'.format(self.name))
+        logger.setLevel(logging.INFO)
+        handler = logging.StreamHandler()
+        fmt = logging.Formatter('%(asctime)s WETEXT %(levelname)s %(message)s')
+        handler.setFormatter(fmt)
+        logger.addHandler(handler)
+
         os.makedirs(cache_dir, exist_ok=True)
         tagger_name = '{}_tagger.fst'.format(prefix)
         verbalizer_name = '{}_verbalizer.fst'.format(prefix)
@@ -88,20 +92,20 @@ class Processor:
         exists = os.path.exists(tagger_path) and os.path.exists(
             verbalizer_path)
         if exists and not overwrite_cache:
-            logging.info("found existing fst: {}".format(tagger_path))
-            logging.info("                    {}".format(verbalizer_path))
-            logging.info("skip building fst for {} ...".format(self.name))
+            logger.info("found existing fst: {}".format(tagger_path))
+            logger.info("                    {}".format(verbalizer_path))
+            logger.info("skip building fst for {} ...".format(self.name))
             self.tagger = Fst.read(tagger_path).optimize()
             self.verbalizer = Fst.read(verbalizer_path).optimize()
         else:
-            logging.info("building fst for {} ...".format(self.name))
+            logger.info("building fst for {} ...".format(self.name))
             self.build_tagger()
             self.build_verbalizer()
             self.tagger.optimize().write(tagger_path)
             self.verbalizer.optimize().write(verbalizer_path)
-            logging.info("done")
-            logging.info("fst path: {}".format(tagger_path))
-            logging.info("          {}".format(verbalizer_path))
+            logger.info("done")
+            logger.info("fst path: {}".format(tagger_path))
+            logger.info("          {}".format(verbalizer_path))
 
     def tag(self, input):
         if len(input) == 0: