WeTextProcessing 1.0.6.dev19__tar.gz → 1.2.0__tar.gz

wetextprocessing-1.2.0/.github/workflows/runtime.yml

@@ -0,0 +1,22 @@
+name: Runtime Build
+
+on:
+  workflow_dispatch:
+  pull_request:
+    paths:
+      - 'runtime/**'
+      - '.github/workflows/runtime.yml'
+
+jobs:
+  build:
+    runs-on: ${{ matrix.os }}
+    strategy:
+      fail-fast: false
+      matrix:
+        os: [ubuntu-latest, macos-latest]
+    steps:
+      - uses: actions/checkout@v4
+      - name: Build runtime
+        run: |
+          cmake -B build -S runtime -DBUILD_TESTING=OFF
+          cmake --build build -j$(nproc 2>/dev/null || sysctl -n hw.ncpu)

{wetextprocessing-1.0.6.dev19 → wetextprocessing-1.2.0}/.github/workflows/unittest.yml

@@ -15,7 +15,7 @@ jobs:
     strategy:
       fail-fast: false
       matrix:
-        os: [ubuntu-latest, macos-latest, windows-latest]
+        os: ${{ github.event_name == 'pull_request' && fromJSON('["ubuntu-latest"]') || fromJSON('["ubuntu-latest", "macos-latest", "windows-latest"]') }}
         python-version: ['3.9']
     steps:
       - uses: actions/checkout@v4

{wetextprocessing-1.0.6.dev19 → wetextprocessing-1.2.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: WeTextProcessing
-Version: 1.0.6.dev19
+Version: 1.2.0
 Summary: WeTextProcessing, including TN & ITN
 Author-email: Zhendong Peng <pzd17@tsinghua.org.cn>, Xingchen Song <sxc19@tsinghua.org.cn>
 Project-URL: Homepage, https://github.com/wenet-e2e/WeTextProcessing

{wetextprocessing-1.0.6.dev19 → wetextprocessing-1.2.0}/WeTextProcessing.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: WeTextProcessing
-Version: 1.0.6.dev19
+Version: 1.2.0
 Summary: WeTextProcessing, including TN & ITN
 Author-email: Zhendong Peng <pzd17@tsinghua.org.cn>, Xingchen Song <sxc19@tsinghua.org.cn>
 Project-URL: Homepage, https://github.com/wenet-e2e/WeTextProcessing

{wetextprocessing-1.0.6.dev19 → wetextprocessing-1.2.0}/WeTextProcessing.egg-info/SOURCES.txt

@@ -6,6 +6,7 @@ CPPLINT.cfg
 LICENSE
 README.md
 pyproject.toml
+.github/workflows/runtime.yml
 .github/workflows/unittest.yml
 .github/workflows/wheels.yml
 WeTextProcessing.egg-info/PKG-INFO
@@ -78,6 +79,63 @@ itn/chinese/test/data/number.txt
 itn/chinese/test/data/time.txt
 itn/chinese/test/data/train_number.txt
 itn/chinese/test/data/whitelist.txt
+itn/english/__init__.py
+itn/english/inverse_normalizer.py
+itn/english/data/currency.tsv
+itn/english/data/date_period.tsv
+itn/english/data/magnitudes.tsv
+itn/english/data/measurements.tsv
+itn/english/data/months.tsv
+itn/english/data/months_cased.tsv
+itn/english/data/whitelist.tsv
+itn/english/data/whitelist_tech.tsv
+itn/english/data/year_suffix.tsv
+itn/english/data/electronic/domain.tsv
+itn/english/data/electronic/server_name.tsv
+itn/english/data/electronic/symbols.tsv
+itn/english/data/electronic/url_symbols.tsv
+itn/english/data/numbers/digit.tsv
+itn/english/data/numbers/hundred.tsv
+itn/english/data/numbers/teen.tsv
+itn/english/data/numbers/thousands.tsv
+itn/english/data/numbers/ties.tsv
+itn/english/data/numbers/zero.tsv
+itn/english/data/ordinals/digit.tsv
+itn/english/data/ordinals/teen.tsv
+itn/english/data/time/minute_to.tsv
+itn/english/data/time/time_suffix.tsv
+itn/english/data/time/time_suffix_cased.tsv
+itn/english/data/time/time_zone.tsv
+itn/english/data/time/time_zone_cased.tsv
+itn/english/data/time/to_hour.tsv
+itn/english/rules/__init__.py
+itn/english/rules/cardinal.py
+itn/english/rules/char.py
+itn/english/rules/date.py
+itn/english/rules/decimal.py
+itn/english/rules/electronic.py
+itn/english/rules/measure.py
+itn/english/rules/money.py
+itn/english/rules/ordinal.py
+itn/english/rules/punctuation.py
+itn/english/rules/telephone.py
+itn/english/rules/time.py
+itn/english/rules/whitelist.py
+itn/english/rules/word.py
+itn/english/test/__init__.py
+itn/english/test/normalizer_test.py
+itn/english/test/utils.py
+itn/english/test/data/en_cardinal.txt
+itn/english/test/data/en_date.txt
+itn/english/test/data/en_decimal.txt
+itn/english/test/data/en_electronic.txt
+itn/english/test/data/en_measure.txt
+itn/english/test/data/en_money.txt
+itn/english/test/data/en_ordinal.txt
+itn/english/test/data/en_telephone.txt
+itn/english/test/data/en_time.txt
+itn/english/test/data/en_whitelist.txt
+itn/english/test/data/en_word.txt
 itn/japanese/__init__.py
 itn/japanese/inverse_normalizer.py
 itn/japanese/data/char/common_chinese_char.tsv
@@ -224,6 +282,7 @@ tn/zh_tn_tagger.fst
 tn/zh_tn_verbalizer.fst
 tn/chinese/__init__.py
 tn/chinese/normalizer.py
+tn/chinese/data/cardinal/phone.tsv
 tn/chinese/data/char/charset_extension.tsv
 tn/chinese/data/char/charset_national_standard_2013_8105.tsv
 tn/chinese/data/char/fullwidth_to_halfwidth.tsv
@@ -237,6 +296,7 @@ tn/chinese/data/default/blacklist.tsv
 tn/chinese/data/default/whitelist.tsv
 tn/chinese/data/erhua/whitelist.tsv
 tn/chinese/data/math/operator.tsv
+tn/chinese/data/math/operator_unambiguous.tsv
 tn/chinese/data/measure/units_en.tsv
 tn/chinese/data/measure/units_zh.tsv
 tn/chinese/data/money/code.tsv
@@ -370,6 +430,7 @@ tn/english/rules/ordinal.py
 tn/english/rules/punctuation.py
 tn/english/rules/range.py
 tn/english/rules/roman.py
+tn/english/rules/serial.py
 tn/english/rules/telephone.py
 tn/english/rules/time.py
 tn/english/rules/whitelist.py

{wetextprocessing-1.0.6.dev19 → wetextprocessing-1.2.0}/itn/chinese/inverse_normalizer.py

@@ -52,19 +52,29 @@ class InverseNormalizer(Processor):
 
     def build_tagger_and_verbalizer(self):
         cardinal = Cardinal(self.convert_number, self.enable_0_to_9, self.enable_million)
+        char = Char()
+        date = Date()
+        fraction = Fraction(cardinal=cardinal)
+        train_number = TrainNumber()
+        math = Math(cardinal=cardinal)
+        measure = Measure(enable_0_to_9=self.enable_0_to_9, cardinal=cardinal)
+        money = Money(enable_0_to_9=self.enable_0_to_9, cardinal=cardinal)
+        time = Time()
+        license_plate = LicensePlate()
+        whitelist = Whitelist()
 
         tagger = (
-            add_weight(Date().tagger, 1.02)
-            | add_weight(Whitelist().tagger, 1.01)
-            | add_weight(Fraction(cardinal=cardinal).tagger, 1.05)
-            | add_weight(Measure(enable_0_to_9=self.enable_0_to_9, cardinal=cardinal).tagger, 1.05)
-            | add_weight(Money(enable_0_to_9=self.enable_0_to_9, cardinal=cardinal).tagger, 1.04)
-            | add_weight(Time().tagger, 1.05)
+            add_weight(date.tagger, 1.02)
+            | add_weight(whitelist.tagger, 1.01)
+            | add_weight(fraction.tagger, 1.05)
+            | add_weight(measure.tagger, 1.05)
+            | add_weight(money.tagger, 1.04)
+            | add_weight(time.tagger, 1.05)
             | add_weight(cardinal.tagger, 1.06)
-            | add_weight(Math(cardinal=cardinal).tagger, 1.10)
-            | add_weight(LicensePlate().tagger, 1.0)
-            | add_weight(TrainNumber().tagger, 1.0)
-            | add_weight(Char().tagger, 100)
+            | add_weight(math.tagger, 1.10)
+            | add_weight(license_plate.tagger, 1.0)
+            | add_weight(train_number.tagger, 1.0)
+            | add_weight(char.tagger, 100)
         ).optimize()
 
         tagger = tagger.star
@@ -72,16 +82,16 @@ class InverseNormalizer(Processor):
 
         verbalizer = (
             cardinal.verbalizer
-            | Char().verbalizer
-            | Date().verbalizer
-            | Fraction().verbalizer
-            | TrainNumber().verbalizer
-            | Math().verbalizer
-            | Measure(enable_0_to_9=self.enable_0_to_9).verbalizer
-            | Money(enable_0_to_9=self.enable_0_to_9).verbalizer
-            | Time().verbalizer
-            | LicensePlate().verbalizer
-            | Whitelist().verbalizer
+            | char.verbalizer
+            | date.verbalizer
+            | fraction.verbalizer
+            | train_number.verbalizer
+            | math.verbalizer
+            | measure.verbalizer
+            | money.verbalizer
+            | time.verbalizer
+            | license_plate.verbalizer
+            | whitelist.verbalizer
         ).optimize()
         postprocessor = PostProcessor(remove_interjections=self.remove_interjections).processor
 

wetextprocessing-1.2.0/itn/english/data/currency.tsv

@@ -0,0 +1,168 @@
+؋	afghan afghani
+l	albanian lek
+دج	algerian dinar
+kz	angolan kwanza
+aed	arab emirates dirham
+$	argentine peso
+֏	armenian dram
+ƒ	aruban florin
+a$	australian dollar
+₼	azerbaijani manat
+b$	bahamian dollar
+.د.ب	bahraini dinar
+৳	bangladeshi taka
+$	barbadian dollar
+br	belarusian ruble
+bz$	belize dollar
+$	bermudian dollar
+nu	bhutanese ngultrum
+₿	bitcoin
+bs	bolivian boliviano
+km	bosnia and herzegovina convertible mark
+p	botswana pula
+r$	brazilian real
+£	british pound
+b$	brunei dollar
+лв	bulgarian lev
+fbu	burundian franc
+៛	cambodian riel
+ca$	canadian dollar
+cve	cape verde escudo
+cve	cape verdean escudo
+ci$	cayman islands dollar
+cfa	cfa franc
+$	chilean peso
+元	chinese yuan
+$	colombian peso
+cf	comorian franc
+fc	congolese franc
+₡	costa rican colón
+kn	croatian kuna
+cuc$	cuban peso
+cyp	cypriot pound
+kč	czech koruna
+d	dalasi
+kr	danish krone
+fdj	djiboutian franc
+$	dollar
+rd$	dominican peso
+$	east caribbean dollar
+e£	egyptian pound
+aed	emirati dirham
+nkf	eritrean nakfa
+Ξ	ethereum
+br	ethiopian birr
+€	euro
+fk£	falkland islands pound
+fim	finnish markka
+₾	georgian lari
+dm	german mark
+gh₵	ghanaian cedi
+q	guatemalan quetzal
+fg	guinean franc
+g$	guyanese dollar
+g	haitian gourde
+l	honduran lempira
+hk$	hong kong dollar
+ft	hungarian forint
+₹	inr
+íkr	icelandic króna
+₹	indian rupee
+rp	indonesian rupiah
+﷼	iranian rial
+ع.د	iraqi dinar
+₪	israeli new shekel
+₪	israeli shekel
+j$	jamaican dollar
+¥	japanese yen
+ينار	jordanian dinar
+₸	kazakhstani tenge
+ksh	kenyan shilling
+ك	kuwaiti dinar
+k	kyat
+som	kyrgyzstani som
+₭	lao kip
+ლარი	lari
+ل.ل	lebanese pound
+l	lesotho loti
+ld$	liberian dollar
+ld	libyan dinar
+l	lilangeni
+ł	litecoin
+mop$	macanese pataca
+ar	malagasy ariary
+k	malawian kwacha
+rm	malaysian ringgit
+rf	maldivian rufiyaa
+mro	mauritanian ouguiya
+₨	mauritian rupee
+$	mexican peso
+l	moldovan leu
+ɱ	monero
+₮	mongolian tögrög
+dh	moroccan dirham
+mt	mozambican metical
+n$	namibian dollar
+rs	nepalese rupee
+ƒ	netherlands antillean guilder
+nt$	new taiwan dollar
+nz$	new zealand dollar
+c$	nicaraguan córdoba
+₦	nigerian naira
+₩	north korean won
+kr	norwegian krone
+ر.ع	omani rial
+um	ouguiya
+pkr	pakistani rupee
+b/.	panamanian balboa
+₲	paraguayan guaraní
+s/.	peruvian sol
+₱	philippine peso
+zł	polish zloty
+£	pounds sterling
+ر.ق	qatari riyal
+£	quid
+r	real
+xrp	ripples
+lei	romanian leu
+₹	rupee
+₽	russian ruble
+r₣	rwandan franc
+shp	saint helena pounds
+db	são tomé and príncipe dobra
+sar	saudi riyal
+ден	second macedonian denar
+rsd	serbian dinar
+scr	seychelles rupee
+le	sierra leonean leone
+s$	singapore dollar
+sh.so.	somali shilling
+tjs	somoni
+r	south african rand
+₩	south korean won
+ss£	south sudanese pound
+රු	sri lankan rupee
+sdg	sudanese pound
+sr$	surinamese dollar
+kr	swedish krona
+chf	swiss franc
+£s	syrian pound
+৳	taka
+tzs	tanzanian shilling
+₮	tether
+฿	thai baht
+tt$	trinidad and tobago dollar
+د.ت	tunisian dinar
+₺	turkish lira
+m	turkmen new manat
+ush	ugandan shilling
+₴	ukrainian hryvna
+$	united states dollar
+$u	uruguayan peso
+$	us dollar
+som	uzbekistan som
+bs.	venezuelan bolívar
+₫	vietnamese đồng
+₩	won
+﷼	yemeni rial
+¥	yen

wetextprocessing-1.2.0/itn/english/data/date_period.tsv

@@ -0,0 +1,10 @@
+Q1	first quarter
+Q1	q one
+Q2	second quarter
+Q2	q two
+Q3	third quarter
+Q3	q three
+Q4	fourth quarter
+Q4	q four
+H1	first half
+H2	second half

wetextprocessing-1.2.0/itn/english/data/electronic/domain.tsv

@@ -0,0 +1,10 @@
+com
+uk
+fr
+net
+br
+in
+ru
+de
+it
+ai

wetextprocessing-1.2.0/itn/english/data/electronic/server_name.tsv

@@ -0,0 +1,17 @@
+g mail	gmail
+gmail
+n vidia	nvidia
+nvidia
+outlook
+hotmail
+yahoo
+aol
+gmx
+msn
+live
+yandex
+orange
+wanadoo
+web
+comcast
+google

wetextprocessing-1.2.0/itn/english/data/electronic/symbols.tsv

@@ -0,0 +1,22 @@
+.	dot
+-	dash
+-	hyphen
+_	underscore
+!	exclamation mark
+#	number sign
+$	dollar sign
+%	percent sign
+&	ampersand
+'	quote
+*	asterisk
++	plus
+/	slash
+=	equal sign
+?	question mark
+^	circumflex
+`	right single quote
+{	left brace
+|	vertical bar
+}	right brace
+~	tilde
+,	comma

wetextprocessing-1.2.0/itn/english/data/electronic/url_symbols.tsv

@@ -0,0 +1,5 @@
+.	dot
+-	dash
+-	hyphen
+_	underscore
+/	slash

wetextprocessing-1.2.0/itn/english/data/magnitudes.tsv

@@ -0,0 +1,4 @@
+k	thousand
+m	million
+b	billion
+t	trillion

wetextprocessing-1.2.0/itn/english/data/measurements.tsv

@@ -0,0 +1,149 @@
+°F	fahrenheit
+°C	celsius
+°F	degree fahrenheit
+°C	degree celsius
+K	kelvin
+km	kilometer
+m	meter
+cm	centimeter
+mm	millimeter
+ha	hectare
+mi	mile
+m²	square meter
+km²	square kilometer
+ft	foot
+%	percent
+hz	hertz
+kW	kilowatt
+kW	kilo watt
+kWh	kilo watt hour
+kWh	kilowatt hour
+Wh	watt hour
+W	watt
+hp	horsepower
+hp	horse power
+mg	milligram
+kg	kilogram
+ghz	gigahertz
+khz	kilohertz
+mhz	megahertz
+v	volt
+h	hour
+mc	mega coulomb
+s	second
+nm	nanometer
+rpm	revolution per minute
+min	minute
+mA	milli ampere
+%	per cent
+m³	cubic meter
+mph	mile per hour
+tW	tera watt
+mv	milli volt
+mW	megawatt
+mW	mega watt
+μm	micrometer
+"	inch
+cc	c c
+g	gram
+da	dalton
+atm	atmosphere
+ω	ohm
+db	decibel
+ps	peta second
+oz	ounce
+hl	hecto liter
+μg	microgram
+pg	petagram
+kb	kilobit
+mb	megabit
+gb	gigabit
+tb	terabit
+pb	petabit
+ev	electron volt
+mb	megabyte
+kb	kilobyte
+gb	gigabyte
+tb	terabyte
+pb	peta byte
+bps	bit per second
+kbps	kilobit per second
+mbps	megabit per second
+gbps	gigabit per second
+kbps	kilo bit per second
+mbps	mega bit per second
+mbps	mega bit per second
+tbps	terabit per second
+tbps	tera bit per second
+pbps	petabit per second
+pbps	peta bit per second
+kb/s	kilobyte per second
+kb/s	kilo byte per second
+mb/s	megabyte  per second
+mb/s	mega byte  per second
+gb/s	gigabyte per second
+gb/s	giga byte per second
+tb/s	terabyte per second
+tb/s	tera byte per second
+pb/s	petabyte per second
+pb/s	peta byte per second
+st	stone
+kl	kilo liter
+tj	tera joule
+kv	kilo volt
+mv	mega volt
+kn	kilo newton
+kn	kilonewton
+mm	megameter
+au	astronomical unit
+yd	yard
+rad	radian
+lm	lumen
+hs	hecto second
+mol	mole
+gpa	giga pascal
+ml	milliliter
+gW	gigawatt
+gW	gigaWatt
+A	ampere
+mA	mili ampere
+µA	micro ampere
+MA	mega ampere
+kt	knot
+kgf	kilogram force
+ng	nano gram
+ns	nanosecond
+ms	mega siemens
+bar	bar
+gl	giga liter
+μs	microsecond
+dA	deci ampere
+pa	pascal
+ds	deci second
+ms	milli second
+dm	deci meter
+dm³	cubic deci meter
+amu	atomic mass unit
+mf	mega farad
+bq	becquerel
+pb	petabit
+mm²	square millimeter
+cm²	square centimeter
+sq mi	square mile
+sq ft	square foot
+kpa	kilopascal
+cd	candela
+tl	tera liter
+ms	mega second
+mpa	megapascal
+pm	peta meter
+gWh	giga watt hour
+kcal	kilo calory
+gy	gray
+sv	sievert
+cwt	hundredweight
+cc	c c
+mph	miles per hour
+sq ft	square feet
+kgf/cm²	kilograms force per square centimeter
+kgf/cm²	kilogram force per square centimeter

wetextprocessing-1.2.0/itn/english/data/months.tsv

@@ -0,0 +1,12 @@
+january
+february
+march
+april
+may
+june
+july
+august
+september
+october
+november
+december

wetextprocessing-1.2.0/itn/english/data/months_cased.tsv

@@ -0,0 +1,12 @@
+January
+February
+March
+April
+May
+June
+July
+August
+September
+October
+November
+December

wetextprocessing-1.2.0/itn/english/data/numbers/digit.tsv

@@ -0,0 +1,9 @@
+one	1
+two	2
+three	3
+four	4
+five	5
+six	6
+seven	7
+eight	8
+nine	9

wetextprocessing-1.2.0/itn/english/data/numbers/hundred.tsv

@@ -0,0 +1 @@
+hundred

wetextprocessing-1.2.0/itn/english/data/numbers/teen.tsv

@@ -0,0 +1,10 @@
+ten	10
+eleven	11
+twelve	12
+thirteen	13
+fourteen	14
+fifteen	15
+sixteen	16
+seventeen	17
+eighteen	18
+nineteen	19

wetextprocessing-1.2.0/itn/english/data/numbers/thousands.tsv

@@ -0,0 +1,22 @@
+thousand
+million
+billion
+trillion
+quadrillion
+quintillion
+sextillion
+septillion
+octillion
+nonillion
+decillion
+undecillion
+duodecillion
+tredecillion
+quattuordecillion
+quindecillion
+sexdecillion
+septendecillion
+octodecillion
+novemdecillion
+vigintillion
+centillion

wetextprocessing-1.2.0/itn/english/data/numbers/ties.tsv

@@ -0,0 +1,9 @@
+twenty	2
+thirty	3
+forty	4
+fourty	4
+fifty	5
+sixty	6
+seventy	7
+eighty	8
+ninety	9